diff --git a/docker/pypi/wmcore-dev/.dockerignore b/docker/pypi/wmcore-dev/.dockerignore new file mode 100644 index 000000000..4efd9c931 --- /dev/null +++ b/docker/pypi/wmcore-dev/.dockerignore @@ -0,0 +1,5 @@ +* +!TestScripts +!ContainerScripts +!entrypoint.sh +!etc diff --git a/docker/pypi/wmcore-dev/.gitignore b/docker/pypi/wmcore-dev/.gitignore new file mode 100644 index 000000000..b1a6de59f --- /dev/null +++ b/docker/pypi/wmcore-dev/.gitignore @@ -0,0 +1,5 @@ +contents/ +ContainerScripts/LatestUnitTests +.DS_Store +*/**/venv +requirements.txt diff --git a/docker/pypi/wmcore-dev/ContainerScripts/AggregatePylint.py b/docker/pypi/wmcore-dev/ContainerScripts/AggregatePylint.py new file mode 100755 index 000000000..84559c69b --- /dev/null +++ b/docker/pypi/wmcore-dev/ContainerScripts/AggregatePylint.py @@ -0,0 +1,95 @@ +#! /usr/bin/env python3 + +import json + +from optparse import OptionParser + +usage = "usage: %prog [options] message" +parser = OptionParser(usage) +(options, args) = parser.parse_args() +if len(args) != 1: + parser.error("You must supply a label") + +label = args[0] + +try: + with open('pylintReport.json', 'r') as reportFile: + report = json.load(reportFile) +except IOError: + report = {} + +warnings = 0 +errors = 0 +comments = 0 +refactors = 0 +score = 0 + +with open('pylint.out', 'r') as pylintFile: + for line in pylintFile: + if line.startswith('Your code has been rated at '): + scorePart = line.strip('Your code has been rated at ') + score = scorePart.split('/')[0] + try: + if not filename in report: + report[filename] = {} + if not label in report[filename]: + report[filename][label] = {} + if filename and label: + report[filename][label]['score'] = score + except NameError: + print("Score of %s found, but no filename" % score) + + parts = line.split(':') + if len(parts) != 3: + continue + try: + newFilename, lineNumber, rawMessage = parts + newFilename = newFilename.strip() + if not newFilename: # Don't update filename if we didn't find one + continue + lineNumber = int(lineNumber) + filename = newFilename + rmParts = rawMessage.split(']', 1) + rawCode = rmParts[0].strip() + message = rmParts[1].strip() + severity = rawCode[1:2] + code = rawCode[2:6] + shortMsg = rawCode[7:] + msgParts = shortMsg.split(',') + objectName = msgParts[1].strip() + + if severity == 'R': + refactors += 1 + elif severity == 'W': + warnings += 1 + elif severity == 'E': + errors += 1 + elif severity == 'C': + comments += 1 + + if not filename in report: + report[filename] = {} + + if not label in report[filename]: + report[filename][label] = {} + if not 'events' in report[filename][label]: + report[filename][label]['events'] = [] + report[filename][label]['events'].append((lineNumber, severity, code, objectName, message)) + + report[filename][label]['refactors'] = refactors + report[filename][label]['warnings'] = warnings + report[filename][label]['errors'] = errors + report[filename][label]['comments'] = comments + + except ValueError: + continue + +with open('pylintReport.json', 'w') as reportFile: + json.dump(report, reportFile, indent=2) + reportFile.write('\n') + + + + + + diff --git a/docker/pypi/wmcore-dev/ContainerScripts/AnalyzePyFuture.py b/docker/pypi/wmcore-dev/ContainerScripts/AnalyzePyFuture.py new file mode 100755 index 000000000..94092147d --- /dev/null +++ b/docker/pypi/wmcore-dev/ContainerScripts/AnalyzePyFuture.py @@ -0,0 +1,20 @@ +#! /usr/bin/env python + +from __future__ import print_function, division + +import os + +with open('addedFiles.txt', 'r') as addedFiles: + for fileName in addedFiles: + fileName = fileName.strip() + if fileName.endswith('__init__.py'): + continue + with open(fileName, 'r') as pyFile: + pyLines = pyFile.readlines() + if fileName.endswith('.py') or 'python' in pyLines[0]: + foundDivision = False + for line in pyLines: + if '__future__' in line and 'division' in line: + foundDivision = True + if not foundDivision: + print ("* New file %s does not use python 3 division. Please add `from __future__ import division`.\n" % fileName) diff --git a/docker/pypi/wmcore-dev/ContainerScripts/CompareTests.py b/docker/pypi/wmcore-dev/ContainerScripts/CompareTests.py new file mode 100755 index 000000000..195e667d7 --- /dev/null +++ b/docker/pypi/wmcore-dev/ContainerScripts/CompareTests.py @@ -0,0 +1,114 @@ +#!/usr/bin/env python + +from __future__ import print_function + +import glob +import os +import sys + +try: + from github import Github +except ImportError: + # DMWM-WMCore-UnitTests and DMWM-WMCorePy3-UnitTests don't really push anything! + Github = None + +import xunitparser + +testResults = {} + +unstableTests = [] + +try: + with open('code/test/etc/UnstableTests.txt') as unstableFile: + for line in unstableFile: + unstableTests.append(line.strip()) +except: + print("Was not able to open list of unstable tests") + +# Parse all the various nose xunit test reports looking for changes +filePattern = '*/nosetestspy3-*.xml' +if len(sys.argv) == 2: + filePattern = "*/%s-*.xml" % sys.argv[1] +for kind, directory in [('base', './MasterUnitTests/'), ('test', './LatestUnitTests/')]: + for xunitFile in glob.iglob(directory + filePattern): + + ts, tr = xunitparser.parse(open(xunitFile)) + for tc in ts: + testName = '%s:%s' % (tc.classname, tc.methodname) + if testName in testResults: + testResults[testName].update({kind: tc.result}) + else: + testResults[testName] = {kind: tc.result} + +# Generate a Github report of any changes found + +issueID, mode = None, None + +if 'ghprbPullId' in os.environ: + issueID = os.environ['ghprbPullId'] + mode = 'PR' +elif 'TargetIssueID' in os.environ: + issueID = os.environ['TargetIssueID'] + mode = 'Daily' + +print("Comparing tests for issueID: {} in mode: {}".format(issueID, mode)) + +message = 'Unit test changes for pull request %s:\n' % issueID +if mode == 'Daily': + message = 'Unit test changes for most recent test of master branch:\n' + +changed = False +stableChanged = False +failed = False +errorConditions = ['error', 'failure'] + +for testName, testResult in sorted(testResults.items()): + if 'base' in testResult and 'test' in testResult and testName in unstableTests: + if testResult['base'] != testResult['test']: + changed = True + message += "* %s (unstable) changed from %s to %s\n" % (testName, testResult['base'], testResult['test']) + elif 'base' in testResult and 'test' in testResult: + if testResult['base'] != testResult['test']: + changed = True + stableChanged = True + message += "* %s changed from %s to %s\n" % (testName, testResult['base'], testResult['test']) + if testResult['test'] in errorConditions: + failed = True + elif 'test' in testResult: + changed = True + stableChanged = True + message += "* %s was added. Status is %s\n" % (testName, testResult['test']) + if testResult['test'] in errorConditions: + failed = True + elif 'base' in testResult: + changed = True + stableChanged = True + message += "* %s was deleted. Prior status was %s\n" % (testName, testResult['base']) +if failed: + message += '\n\nPreviously working unit tests have failed!\n' + +if mode == 'Daily': + # Alan on 25/may/2021: then there is nothing else to be done + print(message) + sys.exit(0) + +gh = Github(os.environ['DMWMBOT_TOKEN']) +codeRepo = os.environ.get('CODE_REPO', 'WMCore') +repoName = '%s/%s' % (os.environ['WMCORE_REPO'], codeRepo) + +issue = gh.get_repo(repoName).get_issue(int(issueID)) + +if not changed and mode == 'Daily': + message = "No changes to unit tests for latest build\n" +elif not changed: + message = "No changes to unit tests for pull request %s\n" % issueID + +if mode == 'Daily' and stableChanged: + issue.create_comment('%s' % message) +elif mode != 'Daily': + issue.create_comment('%s' % message) + +if failed: + print('Testing of python code. DMWM-FAIL-UNIT') +else: + print('Testing of python code. DMWM-SUCCEED-UNIT') diff --git a/docker/pypi/wmcore-dev/ContainerScripts/IdentifyPythonFiles.py b/docker/pypi/wmcore-dev/ContainerScripts/IdentifyPythonFiles.py new file mode 100755 index 000000000..f632f426e --- /dev/null +++ b/docker/pypi/wmcore-dev/ContainerScripts/IdentifyPythonFiles.py @@ -0,0 +1,31 @@ +#! /usr/bin/env python3 + +from __future__ import print_function, division + +import os +from optparse import OptionParser + +usage = "usage: %prog [options] list_of_files.txt" +parser = OptionParser(usage) +(options, args) = parser.parse_args() +if len(args) != 1: + parser.error("You must supply a file with a list of files to check") + +list_of_files = args[0] + +with open(list_of_files, 'r') as changedFiles: + for fileName in changedFiles: + fileName = fileName.strip() + if not fileName: + continue + if fileName.endswith('.py'): + print(fileName) + continue + try: + with open(fileName, 'r') as pyFile: + pyLines = pyFile.readlines() + if 'python' in pyLines[0]: + print(fileName) + continue + except IOError: + pass diff --git a/docker/pypi/wmcore-dev/ContainerScripts/deployWMAgent.sh b/docker/pypi/wmcore-dev/ContainerScripts/deployWMAgent.sh new file mode 100755 index 000000000..d304b98fb --- /dev/null +++ b/docker/pypi/wmcore-dev/ContainerScripts/deployWMAgent.sh @@ -0,0 +1,12 @@ +#! /bin/bash + +. ./cms-bot/DMWM/setup-secrets.sh +. ./cms-bot/DMWM/update-deployment.sh +. ./cms-bot/DMWM/latest-dmwm-versions.sh + +if [ -z "$WMAGENT_VERSION" ]; then + export WMAGENT_VERSION=$WMAGENT_LATEST +fi + +. ./cms-bot/DMWM/deploy-wmagent.sh + diff --git a/docker/pypi/wmcore-dev/ContainerScripts/deploy_unittest_py3.sh b/docker/pypi/wmcore-dev/ContainerScripts/deploy_unittest_py3.sh new file mode 100755 index 000000000..bc65e42a1 --- /dev/null +++ b/docker/pypi/wmcore-dev/ContainerScripts/deploy_unittest_py3.sh @@ -0,0 +1,94 @@ +#!/bin/bash + +### +# usage +# deploy current version of wmagent external library and use the dmwm/master WMCore code to set up unittest +# sh ./deploy_unittest.sh +# +# Also optional values can be specified. -v deploy agent version, -r git repository for code to test, +# -b branch name of the repository. +# following +# i.e) sh ./deploy_unittest.sh -v 1.0.6 -r ticonaan -v 1.0.5_wmagent +# +# for running the test check the tutorial, https://github.com/dmwm/WMCore/wiki/Setup-wmcore-unittest +### +DMWM_ARCH=slc7_amd64_gcc630 +VERSION=$(curl -s "http://cmsrep.cern.ch/cgi-bin/repos/comp/$DMWM_ARCH?C=M;O=D" | grep -oP "(?<=>cms\+wmagentpy3-dev\+).*(?=-1-1)" | head -1) + +REPOSITORY=dmwm +BRANCH= +UPDATE=false + +deploy_agent() { + + git clone https://github.com/dmwm/deployment.git + curl -s https://raw.githubusercontent.com/dmwm/WMCore/master/test/deploy/init.sh > init.sh + curl -s https://raw.githubusercontent.com/dmwm/WMCore/master/test/deploy/env_unittest_py3.sh > env_unittest_py3.sh + curl -s https://raw.githubusercontent.com/dmwm/WMCore/master/test/deploy/WMAgent_unittest.secrets > WMAgent_unittest.secrets + source ./init.sh + # set -e + for step in prep sw post; do + echo -e "\n*** Deploying WMAgent py3: running $step step ***" + $PWD/deployment/Deploy -R wmagentpy3-dev@$1 -r comp=comp -t $1 -A $DMWM_ARCH -s $step $INSTALL_DIR wmagentpy3/devtools + if [ $? -ne 0 ]; then + ls $INSTALL_DIR + cat $INSTALL_DIR/.deploy/*-$step.log + exit 1 + fi + done + # set +e +} + +setup_test_src() { + ( + mkdir $TEST_DIR; + cd $TEST_DIR; + git clone https://github.com/$1/WMCore.git; + cd WMCore + # if branch is set check out the branch + if [ -n $2 ] + then + git checkout $2 + fi; + ) +} + +update_src() { + ( + rm -rf $TEST_DIR; + setup_test_src $1 $2 + ) +} + +while [ $# -gt 0 ] +do + case "$1" in + -v) VERSION=$2; shift;; + -r) REPOSITORY=$2; shift;; + -b) BRANCH=$2; shift;; + -u) UPDATE=true;; + *) break;; # terminate while loop + esac + shift +done + +if [ $UPDATE = "true" ] +then + source ./env_unittest_py3.sh + update_src $REPOSITORY $BRANCH +else + echo "--- deploying agent $VERSION with local user $USER" + # deploy agent + deploy_agent $VERSION + echo "--- updating agent source repository $REPOSITORY, branch $BRANCH" + # checkout test source + setup_test_src $REPOSITORY $BRANCH + + # swap the source code from deployed one test source + source ./env_unittest_py3.sh + echo "--- starting services" + $manage start-services + + echo "--- creating MariaDB database" + mysql -u $USER --socket=$INSTALL_DIR/current/install/mysql/logs/mysql.sock --execute "create database wmcore_unittest" +fi diff --git a/docker/pypi/wmcore-dev/ContainerScripts/fixCertificates.sh b/docker/pypi/wmcore-dev/ContainerScripts/fixCertificates.sh new file mode 100755 index 000000000..7967f144d --- /dev/null +++ b/docker/pypi/wmcore-dev/ContainerScripts/fixCertificates.sh @@ -0,0 +1,10 @@ +#! /bin/bash + +set -x + +# Certificates are either not readable by the container or have the wrong permissions + +cp orig-certs/servicecert.pem.orig certs/servicecert.pem +cp orig-certs/servicekey.pem.orig certs/servicekey.pem +chmod 600 certs/servicecert.pem +chmod 400 certs/servicekey.pem diff --git a/docker/pypi/wmcore-dev/ContainerScripts/installCMSBot.sh b/docker/pypi/wmcore-dev/ContainerScripts/installCMSBot.sh new file mode 100755 index 000000000..701d34698 --- /dev/null +++ b/docker/pypi/wmcore-dev/ContainerScripts/installCMSBot.sh @@ -0,0 +1,9 @@ +git clone https://github.com/cms-sw/cms-bot +pushd cms-bot/ + +wget https://pypi.python.org/packages/source/r/requests/requests-2.3.0.tar.gz#md5=7449ffdc8ec9ac37bbcd286003c80f00 +tar -xvf requests-2.3.0.tar.gz +rm -rf requests || true +mv requests-2.3.0/requests/ requests + +popd diff --git a/docker/pypi/wmcore-dev/ContainerScripts/installWMCore.sh b/docker/pypi/wmcore-dev/ContainerScripts/installWMCore.sh new file mode 100755 index 000000000..b293340de --- /dev/null +++ b/docker/pypi/wmcore-dev/ContainerScripts/installWMCore.sh @@ -0,0 +1,12 @@ +#! /bin/sh + +# Download the script to install everything +curl https://raw.githubusercontent.com/dmwm/WMCore/master/test/deploy/deploy_unittest.sh > /home/dmwm/ContainerScripts/deploy_unittest.sh +chmod +x /home/dmwm/ContainerScripts/deploy_unittest.sh +sh /home/dmwm/ContainerScripts/deploy_unittest.sh + +echo "export PYTHONPATH=/home/dmwm/wmcore_unittest/WMCore/src/python:\$PYTHONPATH" >> ./env_unittest.sh +# Shut down services so the docker container doesn't have stale PID & socket files +source ./env_unittest.sh +$manage stop-services + diff --git a/docker/pypi/wmcore-dev/ContainerScripts/installWMCorePy3.sh b/docker/pypi/wmcore-dev/ContainerScripts/installWMCorePy3.sh new file mode 100755 index 000000000..5074bdd22 --- /dev/null +++ b/docker/pypi/wmcore-dev/ContainerScripts/installWMCorePy3.sh @@ -0,0 +1,12 @@ +#! /bin/sh + +# Download the script to install everything +curl https://raw.githubusercontent.com/dmwm/WMCore/master/test/deploy/deploy_unittest_py3.sh > /home/dmwm/ContainerScripts/deploy_unittest_py3.sh +chmod +x /home/dmwm/ContainerScripts/deploy_unittest_py3.sh +sh /home/dmwm/ContainerScripts/deploy_unittest_py3.sh + +echo "export PYTHONPATH=/home/dmwm/wmcore_unittest/WMCore/src/python:\$PYTHONPATH" >> ./env_unittest_py3.sh +# Shut down services so the docker container doesn't have stale PID & socket files +source ./env_unittest_py3.sh +$manage stop-services + diff --git a/docker/pypi/wmcore-dev/ContainerScripts/pyfutureTest.sh b/docker/pypi/wmcore-dev/ContainerScripts/pyfutureTest.sh new file mode 100755 index 000000000..7cd05fa0a --- /dev/null +++ b/docker/pypi/wmcore-dev/ContainerScripts/pyfutureTest.sh @@ -0,0 +1,40 @@ +#!/usr/bin/env bash + +if [ -z "$ghprbPullId" -o -z "$ghprbTargetBranch" ]; then + echo "Not all necessary environment variables set: ghprbPullId, ghprbTargetBranch" + exit 1 +fi + +echo "$(TZ=GMT date): running pyfutureTest.sh script" +source ./env_unittest.sh + +pushd wmcore_unittest/WMCore +export PYTHONPATH=`pwd`/test/python:`pwd`/src/python:${PYTHONPATH} + +git config remote.origin.url https://github.com/dmwm/WMCore.git +git fetch origin pull/${ghprbPullId}/merge:PR_MERGE +export COMMIT=`git rev-parse "PR_MERGE^{commit}"` +git checkout -f ${COMMIT} + +echo "$(TZ=GMT date): figuring out what are the files that changed" +# Find all the changed files and filter python-only +git diff --name-only ${ghprbTargetBranch}..${COMMIT} > allChangedFiles.txt +${HOME}/ContainerScripts/IdentifyPythonFiles.py allChangedFiles.txt > changedFiles.txt +git diff-tree --name-status -r ${ghprbTargetBranch}..${COMMIT} | egrep "^A" | cut -f 2 > allAddedFiles.txt +${HOME}/ContainerScripts/IdentifyPythonFiles.py allAddedFiles.txt > addedFiles.txt +rm -f allChangedFiles.txt allAddedFiles.txt + +echo "$(TZ=GMT date): running futurize 1st stage and some fixers" +while read name; do + futurize -1 $name >> test.patch + futurize -f execfile -f filter -f raw_input $name >> test.patch || true + futurize -f idioms $name >> idioms.patch || true +done added.message + +cp test.patch idioms.patch added.message ${HOME}/artifacts/ +echo "$(TZ=GMT date): done with all tests and copying files over to artifacts!" +popd diff --git a/docker/pypi/wmcore-dev/ContainerScripts/tagLatestWMCore.sh b/docker/pypi/wmcore-dev/ContainerScripts/tagLatestWMCore.sh new file mode 100755 index 000000000..d1580c060 --- /dev/null +++ b/docker/pypi/wmcore-dev/ContainerScripts/tagLatestWMCore.sh @@ -0,0 +1,29 @@ +#!/usr/bin/env bash + +if [ -z "$DMWMBOT_TOKEN" -o -z "$WMCORE_REPO" -o -z "$CODE_REPO" -o -z "$TAG_PREFIX" ]; then + echo "Not all necessary environment variables set: DMWMBOT_TOKEN, WMCORE_REPO, CODE_REPO" + exit 1 +fi + +pushd $WORKSPACE/WMCore + +set +x +git remote set-url origin https://${DMWMBOT_TOKEN}:x-oauth-basic@github.com/${WMCORE_REPO}/${CODE_REPO} +set -x + +git checkout master +git pull + +# Create a new tag and push it +export TAG=${TAG_PREFIX}_`date "+%Y%m%d%H%M%S"` +git tag ${TAG} +echo Will add tag: $TAG +#git push origin ${TAG} + +# Deletes tags from previous days +export VALID_TAGS=`date "+JENKINS_%Y%m%d"` +git tag -d `git tag | grep JENKINS` +git fetch --tags +#git push --delete origin `git tag | grep JENKINS | grep -Ev ${VALID_TAGS}` || true + +popd diff --git a/docker/pypi/wmcore-dev/ContainerScripts/updateGit.sh b/docker/pypi/wmcore-dev/ContainerScripts/updateGit.sh new file mode 100755 index 000000000..70938df23 --- /dev/null +++ b/docker/pypi/wmcore-dev/ContainerScripts/updateGit.sh @@ -0,0 +1,12 @@ +#! /bin/sh + +pushd /home/dmwm/wmcore_unittest/WMCore/ +git pull +git fetch --quiet --tags https://github.com/dmwm/WMCore.git "+refs/heads/*:refs/remotes/origin/*" +git config remote.origin.url https://github.com/dmwm/WMCore.git +git config --add remote.origin.fetch "+refs/heads/*:refs/remotes/origin/*" +git fetch --quiet --tags https://github.com/dmwm/WMCore.git "+refs/pull/*:refs/remotes/origin/pr/*" + + + +popd diff --git a/docker/pypi/wmcore-dev/Dockerfile b/docker/pypi/wmcore-dev/Dockerfile new file mode 100644 index 000000000..d3642fad0 --- /dev/null +++ b/docker/pypi/wmcore-dev/Dockerfile @@ -0,0 +1,69 @@ +# FROM --platform=linux/amd64 registry.cern.ch/cmsweb/alma-base:alma9-20240305 +FROM --platform=linux/amd64 registry.cern.ch/cmsweb/cmsweb-alma9-base:20240530 + +# Install cern repo +RUN dnf install -y 'dnf-command(config-manager)' && \ + dnf config-manager --add-repo https://linuxsoft.cern.ch/cern/alma/9/devel/x86_64/os + +# Install basic tools +RUN dnf upgrade -y && \ + dnf install -y \ + gcc \ + git \ + vim \ + libcurl-devel \ + python3 \ + python3-devel \ + python3-pip \ + mariadb-connector-c-devel \ + cmake \ + mariadb && \ + dnf clean all + +# Install wlcg repo and gfal2 +RUN dnf install -y http://linuxsoft.cern.ch/wlcg/el9/x86_64/wlcg-repo-1.0.0-1.el9.noarch.rpm && \ + dnf -y install \ + python3-gfal2 \ + python3-gfal2-util \ + gfal2-plugin-http \ + gfal2-plugin-dcap \ + gfal2-plugin-file \ + gfal2-plugin-srm \ + gfal2-plugin-xrootd \ + gfal2-plugin-gridftp \ + gfal2-plugin-sftp && \ + dnf clean all + +# Tools for getting proxies +RUN dnf install -y \ + wlcg-voms-cms \ + voms-clients-cpp && \ + dnf clean all + +WORKDIR /home/cmsbld + +# get requirements.txt and remove gfal2 to use RPM gfal2 +RUN curl -O https://raw.githubusercontent.com/dmwm/WMCore/master/requirements.txt && \ + sed '/gfal2/d' /home/cmsbld/requirements.txt > /home/cmsbld/requirements-dev.txt + +# clone dmwm/WMCore repo and remove gfal2 to use RPM gfal2 +# RUN git clone https://github.com/dmwm/WMCore && \ +# sed '/gfal2/d' /home/cmsbld/WMCore/requirements.txt > /home/cmsbld/WMCore/requirements-dev.txt + +RUN pip3 install --no-cache-dir --upgrade pip setuptools wheel && \ + pip3 install --no-cache-dir --ignore-installed -r /home/cmsbld/requirements-dev.txt +# pip3 install --no-cache-dir --ignore-installed -r /home/cmsbld/WMCore/requirements-dev.txt + +ADD TestScripts /home/cmsbld/TestScripts +ADD ContainerScripts /home/cmsbld/ContainerScripts +ADD etc /home/cmsbld/etc +ADD entrypoint.sh /entrypoint.sh + +ENV X509_HOST_CERT=/data/certs/servicecert.pem +ENV X509_HOST_KEY=/data/certs/servicekey.pem +ENV X509_USER_CERT=/data/certs/servicecert.pem +ENV X509_USER_KEY=/data/certs/servicekey.pem + +ENTRYPOINT ["/entrypoint.sh"] + +CMD ["sleep", "infinity"] \ No newline at end of file diff --git a/docker/pypi/wmcore-dev/TestScripts/env_unittest.sh b/docker/pypi/wmcore-dev/TestScripts/env_unittest.sh new file mode 100644 index 000000000..9ae5ff79a --- /dev/null +++ b/docker/pypi/wmcore-dev/TestScripts/env_unittest.sh @@ -0,0 +1,60 @@ +#!/bin/bash + +export USER=`whoami` +export BASE_DIR=$PWD +export TEST_DIR=$BASE_DIR/wmcore_unittest +export TEST_SRC=$TEST_DIR/WMCore/src +export TEST_SRC_PYTHON=$TEST_SRC/python +export INSTALL_DIR=$BASE_DIR/unittestdeploy/wmagent +export ADMIN_DIR=/data/admin/wmagent +export CERT_DIR=/data/certs + +#export ORG_SRC_PYTHON=$INSTALL_DIR/current/apps/wmagentpy3/lib/python3.8/site-packages/ +#export ORG_SRC_OTHER=$INSTALL_DIR/current/apps/wmagentpy3/data +#export DBSOCK=$INSTALL_DIR/current/install/mysql/logs/mysql.sock + +# TODO: Somewhat redundant in TestScripts/test-wmcorepy3.sh +export DATABASE=mysql://${MDB_USER}:${MDB_PASS}@127.0.0.1/${MDB_UNITTEST_DB} +export COUCHURL=http://unittestagent:passwd@localhost:6994 +export DIALECT=MySQL + +#rm -rf $ORG_SRC_PYTHON/* + +# ln -s $TEST_SRC_PYTHON/WMCore/ $ORG_SRC_PYTHON +# ln -s $TEST_SRC_PYTHON/WMComponent/ $ORG_SRC_PYTHON +# ln -s $TEST_SRC_PYTHON/PSetTweaks/ $ORG_SRC_PYTHON +# ln -s $TEST_SRC_PYTHON/WMQuality/ $ORG_SRC_PYTHON +# ln -s $TEST_SRC_PYTHON/Utils/ $ORG_SRC_PYTHON + +# rm -rf $ORG_SRC_OTHER/* + +# ln -s $TEST_SRC/couchapps/ $ORG_SRC_OTHER +# ln -s $TEST_SRC/css/ $ORG_SRC_OTHER +# ln -s $TEST_SRC/html/ $ORG_SRC_OTHER +# ln -s $TEST_SRC/javascript/ $ORG_SRC_OTHER +# ln -s $TEST_SRC/template/ $ORG_SRC_OTHER + +export WMAGENT_SECRETS_LOCATION=$ADMIN_DIR/WMAgent.secrets +export X509_HOST_CERT=$CERT_DIR/servicecert.pem +export X509_HOST_KEY=$CERT_DIR/servicekey.pem +export X509_USER_CERT=$CERT_DIR/servicecert.pem +export X509_USER_KEY=$CERT_DIR/servicekey.pem + +# export install=$INSTALL_DIR/current/install/wmagentpy3 +# export config=$INSTALL_DIR/current/config/wmagentpy3 +# export manage=$config/manage + +# source $INSTALL_DIR/current/apps/wmagentpy3/etc/profile.d/init.sh +# source $INSTALL_DIR/current/apps/wmcorepy3-devtools/etc/profile.d/init.sh + +### some Rucio setup needed for jenkins and docker unit tests +# fetch the values defined in the secrets file and update rucio.cfg file +export RUCIO_HOME=$BASE_DIR # TODO: Change to specific rucio directory +MATCH_RUCIO_HOST=`cat $WMAGENT_SECRETS_LOCATION | grep RUCIO_HOST | sed s/RUCIO_HOST=//` +MATCH_RUCIO_AUTH=`cat $WMAGENT_SECRETS_LOCATION | grep RUCIO_AUTH | sed s/RUCIO_AUTH=//` +sed "s+^rucio_host.*+rucio_host = $MATCH_RUCIO_HOST+" $RUCIO_HOME/etc/rucio.cfg-temp > $RUCIO_HOME/etc/rucio.cfg +sed "s+^auth_host.*+auth_host = $MATCH_RUCIO_AUTH+" $RUCIO_HOME/etc/rucio.cfg-temp > $RUCIO_HOME/etc/rucio.cfg +echo "Updated RUCIO_HOME file under: $RUCIO_HOME" + +export PYTHONPATH=/home/cmsbld/WMCore/test/python:$PYTHONPATH +export PYTHONPATH=/home/cmsbld/WMCore/src/python:$PYTHONPATH \ No newline at end of file diff --git a/docker/pypi/wmcore-dev/TestScripts/pylint3kWMCore.sh b/docker/pypi/wmcore-dev/TestScripts/pylint3kWMCore.sh new file mode 100755 index 000000000..ecc1072f6 --- /dev/null +++ b/docker/pypi/wmcore-dev/TestScripts/pylint3kWMCore.sh @@ -0,0 +1,17 @@ +#!/usr/bin/env bash + +# Run pylint over the entire WMCore code base for compatibility checks + +source ./env_unittest.sh +pushd wmcore_unittest/WMCore +export PYTHONPATH=`pwd`/test/python:`pwd`/src/python:$PYTHONPATH + +git checkout master +git pull origin + +# Run the python3 compatibility checkers in python +echo "Printing Pylint version" +pylint --version + +# Disable: [W1618(no-absolute-import), ] import missing `from __future__ import absolute_import` +pylint --py3k -j 2 -f parseable -d W1618 src/python/* test/python/* diff --git a/docker/pypi/wmcore-dev/TestScripts/pylintPRTest.sh b/docker/pypi/wmcore-dev/TestScripts/pylintPRTest.sh new file mode 100755 index 000000000..31e0f543d --- /dev/null +++ b/docker/pypi/wmcore-dev/TestScripts/pylintPRTest.sh @@ -0,0 +1,61 @@ +#!/usr/bin/env bash + +WORKDIR=/home/cmsbld + +if [ -z "$ghprbPullId" -o -z "$ghprbTargetBranch" ]; then + echo "Not all necessary environment variables set: ghprbPullId, ghprbTargetBranch" + exit 1 +fi + +echo "Executing Pylint for PR ID $ghprbPullId and target branch $ghprbTargetBranch" + +# Setup the environment +echo "Sourcing a python3 unittest environment" +source $WORKDIR/TestScripts/env_unittest.sh +JSON_FILENAME=pylintpy3Report.json +PEP8_FILENAME=pep8py3.txt + +pushd $WORKDIR/WMCore +export PYTHONPATH=`pwd`/test/python:`pwd`/src/python:$PYTHONPATH + +# Figure out the one commit we are interested in and what happens to the repo if we were to merge it +git config remote.origin.url https://github.com/dmwm/WMCore.git +git fetch origin pull/${ghprbPullId}/merge:PR_MERGE +export COMMIT=`git rev-parse "PR_MERGE^{commit}"` +git checkout ${ghprbTargetBranch} +git pull + +# Which python files changed? +git diff --name-only ${ghprbTargetBranch}..${COMMIT} > allChangedFiles.txt +$WORKDIR/ContainerScripts/IdentifyPythonFiles.py allChangedFiles.txt > changedFiles.txt + +echo "Printing Pylint version" +pylint --version + +# Get pylint report for master +git checkout -f $ghprbTargetBranch +echo "{}" > pylintReport.json +echo "*** Running Pylint on the changed files against the $ghprbTargetBranch" +while read name; do + pylint --evaluation='10.0 - ((float(5 * error + warning) / statement) * 10)' --rcfile standards/.pylintrc --msg-template='{path}:{line}: [{msg_id}({symbol}), {obj}] {msg}' $name > pylint.out || true + $WORKDIR/ContainerScripts/AggregatePylint.py base +done pylint.out || true + $WORKDIR/ContainerScripts/AggregatePylint.py test +done ${PEP8_FILENAME} +cp ${PEP8_FILENAME} $WORKDIR/artifacts/ + +popd diff --git a/docker/pypi/wmcore-dev/TestScripts/pylintWMCore.sh b/docker/pypi/wmcore-dev/TestScripts/pylintWMCore.sh new file mode 100755 index 000000000..ccec534bb --- /dev/null +++ b/docker/pypi/wmcore-dev/TestScripts/pylintWMCore.sh @@ -0,0 +1,37 @@ +#!/usr/bin/env bash + +# Run pylint and pep8 (pycodestyle) over the entire WMCore code base + +# Setup the environment +if [[ -f env_unittest_py3.sh ]] +then + echo "Sourcing a python3 unittest environment" + source env_unittest_py3.sh + OUT_FILENAME=pylintpy3.txt +else + echo "Sourcing a python2 unittest environment" + source env_unittest.sh + OUT_FILENAME=pylint.txt +fi + +pushd wmcore_unittest/WMCore +export PYTHONPATH=`pwd`/test/python:`pwd`/src/python:$PYTHONPATH + +git checkout master +git pull origin + +echo "Printing Pylint version" +pylint --version + +# Run pylint on the whole code base +pylint --rcfile=standards/.pylintrc -j 2 -f parseable src/python/* test/python/* + +# Fix pep8 which has the wrong python executable +echo "#! /usr/bin/env python" > ../pep8 +cat `which pep8` >> ../pep8 +chmod +x ../pep8 + +# Run PEP-8 checker but not in pylint format +pycodestyle --format=default --exclude=test/data,.svn,CVS,.bzr,.hg,.git,__pycache__,.tox. + +cp ${PEP8_FILENAME} ${HOME}/artifacts/ \ No newline at end of file diff --git a/docker/pypi/wmcore-dev/TestScripts/runPath.sh b/docker/pypi/wmcore-dev/TestScripts/runPath.sh new file mode 100755 index 000000000..402a585bd --- /dev/null +++ b/docker/pypi/wmcore-dev/TestScripts/runPath.sh @@ -0,0 +1,41 @@ +#! /bin/bash + +echo Running tests in path $1 +set -x + +# Make sure we the certs we use are readable by us and only us + +/home/dmwm/ContainerScripts/fixCertificates.sh + +# Start up services (Couch and MySQL) + +source ./env_unittest.sh +$manage start-services + +cd /home/dmwm/wmcore_unittest/WMCore/ + +# Make sure we base our tests on the latest Jenkins-tested master + +/home/dmwm/ContainerScripts/updateGit.sh + +export LATEST_TAG=`git tag |grep JENKINS| sort | tail -1` +if [ -z "$2" ]; then + export COMMIT=$LATEST_TAG +else + export COMMIT=`git rev-parse "origin/pr/$3/merge^{commit}"` +fi + +# First try to merge this PR into the same tag used for the baseline +# Finally give up and just test the tip of the branch +(git checkout $LATEST_TAG && git merge $COMMIT) || git checkout -f $COMMIT + +# Run tests and watchdog to shut it down if needed + +cd /home/dmwm/wmcore_unittest/WMCore/ +rm test/python/WMCore_t/REST_t/*_t.py +/home/dmwm/cms-bot/DMWM/TestWatchdog.py & +python setup.py test --buildBotMode=true --reallyDeleteMyDatabaseAfterEveryTest=true --testCertainPath=$1 --testTotalSlices=1 --testCurrentSlice=0 + +# Save the results + +cp nosetests.xml /home/dmwm/artifacts/nosetests-path.xml diff --git a/docker/pypi/wmcore-dev/TestScripts/runSlice.sh b/docker/pypi/wmcore-dev/TestScripts/runSlice.sh new file mode 100755 index 000000000..641a3a10b --- /dev/null +++ b/docker/pypi/wmcore-dev/TestScripts/runSlice.sh @@ -0,0 +1,50 @@ +#! /bin/bash + +if [ -z "$1" -o -z "$2" ]; then + echo "Not all necessary environment variables set: Two parameters for slice and number of slices" + exit 1 +fi + +echo Running slice $1 of $2 + +# Make sure we the certs we use are readable by us and only us + +/home/dmwm/ContainerScripts/fixCertificates.sh + +# Start up services (Couch and MySQL) + +source ./env_unittest.sh +$manage start-services + +pushd /home/dmwm/wmcore_unittest/WMCore/ + +# Make sure we base our tests on the latest Jenkins-tested master +# sometimes GitHub has issues, so try each command twice + +timeout -s 9 5m git fetch --tags || timeout -s 9 5m git fetch --tags +timeout -s 9 5m git pull || timeout -s 9 5m git fetch --tags +export LATEST_TAG=`git tag |grep JENKINS| sort | tail -1` + +# Find the commit that represents the tip of the PR the latest tag +if [ -z "$ghprbPullId" ]; then + export COMMIT=$LATEST_TAG +else + git config remote.origin.url https://github.com/dmwm/WMCore.git + timeout -s 9 5m git fetch origin pull/${ghprbPullId}/merge:PR_MERGE + export COMMIT=`git rev-parse "PR_MERGE^{commit}"` +fi + +# First try to merge this PR into the same tag used for the baseline +# If it doesn't merge, just test the tip of the branch +(git checkout $LATEST_TAG && git merge $COMMIT) || git checkout -f $COMMIT + +# Run tests and watchdog to shut it down if needed +export USER=`whoami` +/home/dmwm/cms-bot/DMWM/TestWatchdog.py & +timeout 80m python setup.py test --buildBotMode=true --reallyDeleteMyDatabaseAfterEveryTest=true --testCertainPath=test/python --testTotalSlices=$2 --testCurrentSlice=$1 + +# Save the results + +cp nosetests.xml /home/dmwm/artifacts/nosetests-$1.xml + +popd diff --git a/docker/pypi/wmcore-dev/TestScripts/test-wmcorepy3.sh b/docker/pypi/wmcore-dev/TestScripts/test-wmcorepy3.sh new file mode 100755 index 000000000..7609da909 --- /dev/null +++ b/docker/pypi/wmcore-dev/TestScripts/test-wmcorepy3.sh @@ -0,0 +1,101 @@ +#! /bin/bash -e + +WORKDIR=/home/cmsbld +CODE=$WORKDIR/WMCore + +pushd $WORKDIR + +start=`date +%s` + +set +x +# load environment +echo "Sourcing a python3 unittest environment" +. $WORKDIR/TestScripts/env_unittest.sh + +# load wmagent secrets +. /data/admin/wmagent/WMAgent.secrets + +# This is what the production wmagent-mariadb image defines (via WMA_DATABASE) has as of 2024/07/15 +# TODO: We should either make an option in the mariadb image to define the default WMA_DATABASE or make a custom test mariadb image +MDB_UNITTEST_DB=wmagent + +export DATABASE=mysql://${MDB_USER}:${MDB_PASS}@127.0.0.1/${MDB_UNITTEST_DB} +export COUCHURL="http://${COUCH_USER}:${COUCH_PASS}@${COUCH_HOST}:${COUCH_PORT}" + +# ensure db exists +# retry if fails + + +for i in 1 2 3 4 5; do + mysql -u ${MDB_USER} -h 127.0.0.1 -p${MDB_PASS} --execute "CREATE DATABASE IF NOT EXISTS ${MDB_UNITTEST_DB}" && break + + echo "Attempt $i failed. Trying again..." + sleep 5 +done + +# rucio +export RUCIO_HOST=$RUCIO_HOST +export RUCIO_AUTH=$RUCIO_AUTH +set -x + +pushd $CODE + +git pull origin master + +# use ghprbPullId if triggered from a PR +if [[ ! -z "${ghprbPullId}" ]]; then + git fetch --tags https://github.com/dmwm/WMCore.git "+refs/heads/*:refs/remotes/origin/*" + git config remote.origin.url https://github.com/dmwm/WMCore.git + git config --add remote.origin.fetch "+refs/heads/*:refs/remotes/origin/*" + git fetch --tags --quiet https://github.com/dmwm/WMCore.git "+refs/pull/*:refs/remotes/origin/pr/*" + export COMMIT=`git rev-parse "origin/pr/$ghprbPullId/merge^{commit}"` + export LATEST_TAG=`git tag |grep JENKINS| sort | tail -1` + + # First try to merge this PR into the same tag used for the baseline + # Next try to merge this tag onto current master + # Finally give up and just test the tip of the branch + (git checkout $LATEST_TAG && git merge $COMMIT) || (git checkout master && git merge $COMMIT) || git checkout -f $COMMIT +fi + +popd + +### Some tweaks for the nose run (in practice, there is nothing to change in setup_test.py...) +# working dir includes entire python source - ignore +perl -p -i -e 's/--cover-inclusive//' $CODE/setup_test.py +# cover branches but not external python modules +perl -p -i -e 's/--cover-inclusive/--cover-branches/' $CODE/setup_test.py +perl -p -i -e "s/'--cover-html',//" $CODE/setup_test.py + +#export NOSE_EXCLUDE='AlertGenerator' +# timeout tests after 5 mins +#export NOSE_PROCESSES=1 +#export NOSE_PROCESS_TIMEOUT=300 +#export NOSE_PROCESS_RESTARTWORKER=1 + +# include FWCore.ParameterSet.Config + +export PYTHONPATH=/var/lib/jenkins/additional-library:$PYTHONPATH + +# remove old coverage data +coverage erase + +# debugging python interpreters +echo "Python version is: " && python --version || true +echo "Python3 version is: " && python3 --version || true + +# run test - force success though - failure stops coverage report +rm nosetests*.xml || true +# FIXME Alan on 25/may/2021: ImportError: cannot import name _psutil_linux +#python3 cms-bot/DMWM/TestWatchdog.py & + +python3 $CODE/setup.py test --buildBotMode=true --reallyDeleteMyDatabaseAfterEveryTest=true --testCertainPath=$CODE/test/python --testTotalSlices=$SLICES --testCurrentSlice=$SLICE || true #--testCertainPath=test/python/WMCore_t/WMBS_t || true +mv nosetests.xml artifacts/nosetestspy3-$SLICE-$BUILD_ID.xml + +# Add these here as they need the same environment as the main run +#FIXME: change so initial coverage command skips external code +# coverage xml -i --include=$PWD/install* || true + +end=`date +%s` +runtime=$((end-start)) + +echo "Total time to test slice $SLICE: $runtime" diff --git a/docker/pypi/wmcore-dev/build.sh b/docker/pypi/wmcore-dev/build.sh new file mode 100755 index 000000000..1b6bd5c8a --- /dev/null +++ b/docker/pypi/wmcore-dev/build.sh @@ -0,0 +1,3 @@ +#!/bin/bash + +docker build --tag wmcore-dev . \ No newline at end of file diff --git a/docker/pypi/wmcore-dev/entrypoint.sh b/docker/pypi/wmcore-dev/entrypoint.sh new file mode 100755 index 000000000..e8195c17a --- /dev/null +++ b/docker/pypi/wmcore-dev/entrypoint.sh @@ -0,0 +1,24 @@ +#!/bin/bash + +# add WMCore repository, if in Jenkins +if [[ -z "${BUILD_ID}" ]]; then + echo "No BUILD_ID set" +else + echo "BUILD_ID set, cloning dmwm/WMCore" + git clone https://github.com/dmwm/WMCore + + # give proper permissions to home directory + chown -R ${MY_ID}:${MY_GROUP} /home/cmsbld + + USERN=$(id -un ${MY_ID}) + + su - $USERN +fi + +pushd /home/cmsbld + +# clone jenkins-test scripts +git clone https://github.com/d-ylee/jenkins-test +popd + +$@ \ No newline at end of file diff --git a/docker/pypi/wmcore-dev/etc/rucio.cfg-temp b/docker/pypi/wmcore-dev/etc/rucio.cfg-temp new file mode 100644 index 000000000..b3d3f3a92 --- /dev/null +++ b/docker/pypi/wmcore-dev/etc/rucio.cfg-temp @@ -0,0 +1,20 @@ +# Copyright European Organization for Nuclear Research (CERN) +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# You may not use this file except in compliance with the License. +# You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 +# +# Authors: +# - Vincent Garonne, , 2017 +# - Eric Vaandering, , 2018 + +[common] +[client] +rucio_host = http://cms-rucio-int.cern.ch +auth_host = https://cms-rucio-auth-int.cern.ch +auth_type = x509 +ca_cert = /etc/grid-security/certificates/ +client_cert = $X509_USER_CERT +client_key = $X509_USER_KEY +client_x509_proxy = $X509_USER_PROXY +request_retries = 3 diff --git a/docker/pypi/wmcore-dev/push.sh b/docker/pypi/wmcore-dev/push.sh new file mode 100755 index 000000000..cce5f764a --- /dev/null +++ b/docker/pypi/wmcore-dev/push.sh @@ -0,0 +1,4 @@ +#!/bin/bash + +docker tag wmcore-dev registry.cern.ch/cmsweb/wmcore-dev:latest +docker push registry.cern.ch/cmsweb/wmcore-dev:latest