PyPI packaging for ReqMgr2

MANIFEST.in

@@ -1,4 +1,6 @@
-include setup_*.py
+include setup_build.py
+include setup_dependencies.py
+recursive-include bin *
 include requirements.txt
 include LICENSE
 include NOTICE

requirements.txt

@@ -1,32 +1,35 @@
-# All dependencies needed to run WMAgent
-Cheetah==2.4.0
-CherryPy==17.4.0
-Markdown==3.0.1
-MySQL-python==1.2.5
-SQLAlchemy==1.3.3
-Sphinx==1.3.5
-cx-Oracle==5.2.1
-dbs-client==3.3.160
-decorator==3.4.2
-future==0.18.2
-nose2==0.9.2
-mock==2.0.0
-coverage==4.5.4
-funcsigs==1.0.2
-httplib2==0.18.0
-psutil==5.6.6
-py==1.7.0
-pyOpenSSL==18.0.0
-pycurl-client==3.3.160
-pycurl==7.19.3
-python-cjson==1.2.1
-pyzmq==17.1.2
-retry==0.9.1
-setuptools==39.2.0
-stomp.py==4.1.15
-rucio-clients==1.23.0
-CMSMonitoring>=0.3.4
-# All dependencies needed to run MicroServices
-pymongo==3.10.1
-# All dependencies needed to run Global WorkQueue
-# All dependencies needed to run ReqMgr2
+# All dependencies needed to run WMCore software
+# This file is parsed by tools/build_pypi_packages.sh to create requirements.txt
+# files for each piece of software built from WMCore and uploaded to PyPI
+# Format:
+# PackageName==X.Y.Z          # <comma separated list of WMCore software needing the package>
+
+Cheetah==2.4.0                # wmagent,reqmgr2
+CherryPy==17.4.0              # wmagent,reqmgr2
+Markdown==3.0.1               # wmagent
+MySQL-python==1.2.5           # wmagent
+SQLAlchemy==1.3.3             # wmagent
+Sphinx==1.3.5                 # wmagent,reqmgr2
+cx-Oracle==5.2.1              # wmagent
+dbs-client==3.7.8             # wmagent,reqmgr2
+decorator==3.4.2              # wmagent
+future==0.18.2                # wmagent,reqmgr2
+nose2==0.9.2                  # wmagent
+mock==2.0.0                   # wmagent
+coverage==4.5.4               # wmagent
+funcsigs==1.0.2               # wmagent
+httplib2==0.18.0              # wmagent,reqmgr2
+psutil==5.6.6                 # wmagent,reqmgr2
+py==1.7.0                     # wmagent
+pyOpenSSL==18.0.0             # wmagent
+pycurl-client==3.3.160        # wmagent
+pycurl==7.19.3                # wmagent,reqmgr2
+python-cjson==1.2.1           # wmagent
+pyzmq==17.1.2                 # wmagent
+retry==0.9.1                  # wmagent,reqmgr2
+setuptools==39.2.0            # wmagent
+stomp.py==4.1.15              # wmagent
+rucio-clients==1.23.0         # wmagent
+CMSMonitoring>=0.3.4          # wmagent
+pymongo==3.10.1               # reqmgr2ms
+

setup_build.py

@@ -140,13 +140,17 @@ def check_system(command):
         print(msg)
         sys.exit(1)
 
-def things_to_build(command):
+def things_to_build(command, pypi=False):
     """
     Take a build/install command and determine all the packages and modules it needs to build/install. Modules are
     explicitly listed in the dependancies but packages needs to be generated (to pick up sub directories).
     """
     # work out all the dependent packages
-    dependency_tree = walk_dep_tree(dependencies[command.system])
+    if pypi:
+        dependency_tree = walk_dep_tree(dependencies[command])
+    else:
+        dependency_tree = walk_dep_tree(dependencies[command.system])
+
     # and the corresponding source directories and files
     package_src_dirs = []
     module_src_files = []

setup_dependencies.py

@@ -52,6 +52,22 @@
                     'src/templates/WMCore/WebTools',
                     'src/templates/WMCore/WebTools/Masthead', ]
     },
+    'wmcore': {
+        'packages': ['WMCore+',
+                     'WMComponent+',
+                     'WMQuality+',
+                     'PSetTweaks+',
+                     'Utils+'],
+        'modules': [],
+        'systems': [],
+        'statics': ['src/couchapps+',
+                    'src/css+',
+                    'src/html+',
+                    'src/javascript+',
+                    'src/templates+',
+                    'etc+'
+                    ],
+    },
     'reqmgr2': {
         'packages': ['WMCore.ReqMgr+',
                      'WMCore.WMDataMining+',

setup_template.py

@@ -0,0 +1,65 @@
+#!/usr/bin/env python
+
+# This template for the setup script is used to build several pypi packages
+# from the WMCore codebase. The variable package_name controls which package
+# is built. PACKAGE_TO_BUILD is manipulated via tools/build_pypi_packages.sh
+# at build time.
+#
+# The version number comes from WMCore/__init__.py and needs to
+# follow PEP 440 conventions
+
+from __future__ import print_function, division
+import os
+import sys
+from setuptools import setup, Command
+from setup_build import list_static_files, things_to_build
+from setup_dependencies import dependencies
+
+# get the WMCore version (thanks rucio devs)
+sys.path.insert(0, os.path.abspath('src/python'))
+from WMCore import __version__
+wmcore_version = __version__
+
+# the contents of package_name are modified via tools/build_pypi_packages.sh
+package_name = "PACKAGE_TO_BUILD"
+packages, py_modules = things_to_build(package_name, pypi=True)
+data_files = list_static_files(dependencies[package_name])
+
+# we need to override 'clean' command to remove specific files
+class CleanCommand(Command):
+    user_options = []
+    def initialize_options(self):
+        pass
+    def finalize_options(self):
+        pass
+    def run(self):
+        os.system ('rm -rfv ./dist ./src/python/*.egg-info')
+
+def parse_requirements(requirements_file):
+    """
+      Create a list for the 'install_requires' component of the setup function
+      by parsing a requirements file
+    """
+
+    if os.path.exists(requirements_file):
+        # return a list that contains each line of the requirements file
+        return open(requirements_file, 'r').read().splitlines()
+    else:
+        print("ERROR: requirements file " + requirements_file + " not found.")
+        sys.exit(1)
+
+setup(name=package_name,
+      version=wmcore_version,
+      package_dir={'': 'src/python/'},
+      packages=packages,
+      py_modules=py_modules,
+      data_files=data_files,
+      install_requires=parse_requirements("requirements.txt"),
+      maintainer='CMS DMWM Group',
+      maintainer_email='[email protected]',
+      cmdclass={
+          'clean': CleanCommand,
+      },
+      url="https://github.com/dmwm/WMCore",
+      license="Apache License, Version 2.0",
+      )

tools/build_pypi_packages.sh

@@ -9,7 +9,7 @@
 
 wmcore=false
 wmagent=false
-
+reqmgr2=false
 TOBUILD=$1
 
 case $TOBUILD in
@@ -21,37 +21,49 @@ case $TOBUILD in
       echo "Building wmcore package"
       wmcore=true
       ;;
+    reqmgr2)
+      echo "Building wmcore package"
+      reqmgr2=true
+      ;;
     all)
       echo "Building wmagent package"
       echo "Building wmcore package"
       wmcore=true
       wmagent=true
+      reqmgr2=true
       ;;
     *)
       echo "Please enter one of the following arguments."
       echo "  all       Build all WMCore packages"
       echo "  wmagent   Build wmagent package"
       echo "  wmcore    Build wmcore package"
+      echo "  reqmgr2   Build reqmgr2 package"
       exit 1
 esac
 
-# clean up any previous builds
-rm -fv dist/*
+# make a copy of requirements.txt to reference for each build
+cp requirements.txt requirements.wmcore.txt
 
 if $wmagent
   then
-  /bin/cp requirements.wmagent.txt requirements.txt
-  /bin/cp setup_wmagent.py setup.py
-  python setup.py sdist
-  twine upload dist/wmagent-*
+    sed 's/PACKAGE_TO_BUILD/wmagent/' setup_template.py > setup.py
+    awk '/wmagent/ {print $1}' requirements.wmcore.txt > requirements.txt
+    python setup.py clean sdist
+    twine upload dist/wmagent-*
 fi
 
 if $wmcore
   then
-  /bin/cp requirements.wmcore.txt requirements.txt
-  /bin/cp setup_wmcore.py setup.py
-  python setup.py sdist
-  twine upload dist/wmcore-*
+    sed 's/PACKAGE_TO_BUILD/wmcore/' setup_template.py > setup.py
+    awk '/wmcore/ {print $1}' requirements.wmcore.txt > requirements.txt
+    python setup.py clean sdist
+    twine upload dist/wmcore-*
 fi
 
-
+if $reqmgr2
+  then
+    sed 's/PACKAGE_TO_BUILD/reqmgr2/' setup_template.py > setup.py
+    awk '/reqmgr2/ {print $1}' requirements.wmcore.txt > requirements.txt
+    python setup.py clean sdist
+    twine upload dist/reqmgr2-*
+fi