jenkins-admin.py

#!/usr/bin/env python3
# pylint: disable=invalid-name
#
# Licensed to the Apache Software Foundation (ASF) under one
# or more contributor license agreements.  See the NOTICE file
# distributed with this work for additional information
# regarding copyright ownership.  The ASF licenses this file
# to you under the Apache License, Version 2.0 (the
# "License"); you may not use this file except in compliance
# with the License.  You may obtain a copy of the License at
#
#   http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing,
# software distributed under the License is distributed on an
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
# KIND, either express or implied.  See the License for the
# specific language governing permissions and limitations
# under the License.
#
""" Process patch file attachments from JIRA using a query """

from argparse import ArgumentParser
from tempfile import NamedTemporaryFile
from xml.etree import ElementTree
import os
import re
import requests
import sys


def http_get(resource, ignore_error=False, username=None, password=None):
    """ get the contents of a URL """

    try:
        if username and password:
            if username and password:
                response = requests.get(resource, auth=(username,password))
            else:
                response = requests.get(resource)
        response.raise_for_status()
    except requests.exceptions as http_err: # pylint:
        errstr=str(http_err)
        print(f'%{resource} returns HTTP error %{response.status_code}: %{errstr}\n')
        if ignore_error:
            return ''
        print('Aborting.')
        sys.exit(1)
        raise http_err
    return response.text


def parse_jira_data(filename):
    """ returns a map of (project, issue) => attachment id """
    tree = ElementTree.parse(filename)
    root = tree.getroot()
    jirapattern = re.compile(r'([A-Z]+)\-([0-9]+)')
    result = {}
    for item in root.findall('./channel/item'):
        jirakey = item.find('key')
        if jirakey is None:
            continue
        jiraissue = jirakey.text
        matcher = jirapattern.match(jiraissue)
        if not matcher:
            continue
        jiraissue = (matcher.group(1), matcher.group(2))
        attachmentids = []
        for jiraattachment in item.findall('./attachments/attachment'):
            attachmentid = jiraattachment.get('id')
            try:
                attachmentids.append(int(attachmentid))
            except ValueError:
                pass
        if attachmentids:
            attachmentids.sort()
            result[jiraissue] = attachmentids[-1]
    return result


def main():  #pylint: disable=too-many-branches, too-many-statements, too-many-locals
    """ main program """
    parser = ArgumentParser(prog='jenkins-admin')
    if os.getenv('JENKINS_URL'):
        parser.set_defaults(jenkinsurl=os.getenv('JENKINS_URL'))
    if os.getenv('JOB_NAME'):
        parser.set_defaults(jenkinsJobName=os.getenv('JOB_NAME'))
    else:
        parser.set_defaults(jenkinsJobName='PreCommit-Admin')

    parser.set_defaults(jenkinsJobTemplate='PreCommit-{project}')
    parser.add_argument('--initialize',
                        action='store_true',
                        dest='jenkinsInit',
                        help='Start a new patch_tested.txt file')
    parser.add_argument('--jenkins-jobname',
                        type=str,
                        dest='jenkinsJobName',
                        help='PreCommit-Admin JobName',
                        metavar='JOB_NAME')
    parser.add_argument('--jenkins-project-template',
                        type=str,
                        dest='jenkinsJobTemplate',
                        help='Template for project jobs',
                        metavar='TEMPLATE')
    parser.add_argument('--jenkins-token',
                        type=str,
                        dest='jenkinsToken',
                        help='Jenkins Token',
                        metavar='TOKEN')
    parser.add_argument('--jenkins-url',
                        type=str,
                        dest='jenkinsurl',
                        help='Jenkins base URL',
                        metavar='URL')
    parser.add_argument(
        '--jenkins-url-override',
        type=str,
        dest='jenkinsurloverrides',
        action='append',
        help='Project specific Jenkins base URL',
        metavar='PROJECT=URL',
    )
    parser.add_argument('--jira-filter',
                        type=str,
                        dest='jiraFilter',
                        help='JIRA filter URL',
                        metavar='URL')
    parser.add_argument('--jira-user',
                        type=str,
                        dest='jiraUser',
                        help='JIRA username')
    parser.add_argument('--jira-password',
                        type=str,
                        dest='jiraPassword',
                        help='JIRA password')
    parser.add_argument('--live',
                        dest='live',
                        action='store_true',
                        help='Submit Job to jenkins')
    parser.add_argument('--max-history',
                        dest='history',
                        type=int,
                        help='Maximum history to store',
                        default=5000)
    parser.add_argument(
        '-V',
        '--version',
        dest='release_version',
        action='store_true',
        default=False,
        help="display version information for jenkins-admin and exit.")

    options = parser.parse_args()

    # Handle the version string right away and exit
    if options.release_version:
        with open(os.path.join(os.path.dirname(__file__), "../../VERSION"),
                  'r') as ver_file:
            print(ver_file.read())
        sys.exit(0)

    token_frag = ''
    if options.jenkinsToken:
        token_frag = 'token=%s' % options.jenkinsToken
    else:
        token_frag = 'token={project}-token'
    if not options.jiraFilter:
        parser.error('ERROR: --jira-filter is a required argument.')
    if not options.jenkinsurl:
        parser.error(
            'ERROR: --jenkins-url or the JENKINS_URL environment variable is required.'
        )
    if options.history < 0:
        parser.error('ERROR: --max-history must be 0 or a positive integer.')
    jenkinsurloverrides = {}
    if options.jenkinsurloverrides:
        for override in options.jenkinsurloverrides:
            if '=' not in override:
                parser.error('Invalid Jenkins Url Override: ' + override)
            (project, url) = override.split('=', 1)
            jenkinsurloverrides[project.upper()] = url
    tempfile = NamedTemporaryFile(delete=False)
    try:
        jobloghistory = None
        if not options.jenkinsInit:
            jobloghistory = http_get(
                options.jenkinsurl +
                '/job/%s/lastSuccessfulBuild/artifact/patch_tested.txt' %
                options.jenkinsJobName, True)

            # if we don't have a successful build available try the last build

            if not jobloghistory:
                jobloghistory = http_get(
                    options.jenkinsurl +
                    '/job/%s/lastCompletedBuild/artifact/patch_tested.txt' %
                    options.jenkinsJobName)
            jobloghistory = jobloghistory.strip().split('\n')
            if 'TESTED ISSUES' not in jobloghistory[0]:
                print(
                    'Downloaded patch_tested.txt control file may be corrupted. Failing.'
                )
                sys.exit(1)

        # we are either going to write a new one or rewrite the old one

        joblog = open('patch_tested.txt', 'w+')

        if jobloghistory:
            if len(jobloghistory) > options.history:
                jobloghistory = [jobloghistory[0]] \
                                 + jobloghistory[len(jobloghistory) \
                                 - options.history:]
            for jobhistoryrecord in jobloghistory:
                joblog.write(jobhistoryrecord + '\n')
        else:
            joblog.write('TESTED ISSUES\n')
        joblog.flush()
        rssdata = http_get(options.jiraFilter, False, options.jiraUser,
                           options.jiraPassword)
        tempfile.write(rssdata.encode('utf-8'))
        tempfile.flush()
        for (key, attachment) in list(parse_jira_data(tempfile.name).items()):
            (project, issue) = key
            if jenkinsurloverrides.get(project):
                url = jenkinsurloverrides[project]
            else:
                url = options.jenkinsurl

            jenkinsurltemplate = url + '/job/' \
                + options.jenkinsJobTemplate \
                + '/buildWithParameters?' + token_frag \
                + '&ISSUE_NUM={issue}&ATTACHMENT_ID={attachment}'

            url_args = {
                'project': project,
                'issue': issue,
                'attachment': attachment,
            }
            jenkinsurl = jenkinsurltemplate.format(**url_args)

            # submit job

            jobname = '%s-%s,%s' % (project, issue, attachment)
            if not jobloghistory or jobname not in jobloghistory:
                print(jobname + ' has not been processed, submitting')
                joblog.write(jobname + '\n')
                joblog.flush()
                if options.live:
                    http_get(jenkinsurl, True)
                else:
                    print('GET ' + jenkinsurl)
            else:
                print(jobname + ' has been processed, ignoring')
        joblog.close()
    finally:
        if options.live:
            os.remove(tempfile.name)
        else:
            print('JIRA Data is located: ' + tempfile.name)


if __name__ == '__main__':
    main()