master.cfg

# -*- python -*-
# ex: set syntax=python:
#
# Buildbot config for the Open Lighting Project.
# If you want to add slaves edit 'build.config'.
# The config is organized bottom-up, i.e. slaves, then builders, then
# schedulers.

from buildbot.buildslave import BuildSlave
from buildbot.changes import filter
from buildbot.changes.gitpoller import GitPoller
from buildbot.config import BuilderConfig
from buildbot.process.factory import BuildFactory
from buildbot.scheduler import Try_Userpass
from buildbot.schedulers.basic import SingleBranchScheduler
from buildbot.schedulers.forcesched import ForceScheduler, FixedParameter
from buildbot.status import html
from buildbot.status import words
from buildbot.status.web import authz, auth
from buildbot.steps.shell import Compile
from buildbot.steps.shell import Configure
from buildbot.steps.shell import ShellCommand
from buildbot.steps.source.git import Git
import config_helper

config = config_helper.LoadConfig('build.config')
slaves = config_helper.SlaveStore(config['SLAVES'])

builder_branches = {'ola': {}, 'ja_rule': {}}


def BuildRepoURL(repo):
  """Build a repo URL from it's constituent parts."""
  return "%s%s%s" % (config['BASE_URL'], repo, config['REPO_STUB'])


# This is the dictionary that the buildmaster pays attention to. We also use
# a shorter alias to save typing.
c = BuildmasterConfig = {}

################################################################################
# PROJECT IDENTITY

c['title'] = config['TITLE']
c['titleURL'] = config['URL']
c['buildbotURL'] = config['BUILDBOTURL']

################################################################################
# CHANGESOURCES

# The 'change_source' setting tells the buildmaster how it should find out
# about source code changes.

c['change_source'] = []

ola_branches = [config['OLA_TRUNK_NAME']]
ola_branches.extend(config['OLA_EXTRA_BRANCHES'])
for branch in ola_branches:
  builder_branches['ola'][branch] = []
c['change_source'].append(
    GitPoller(BuildRepoURL(config['OLA_REPO']),
              workdir='gitpoller-workdir',
              branches=ola_branches,
              pollinterval=300,
              project='ola')
)

ja_rule_branches = [config['JA_RULE_TRUNK_NAME']]
ja_rule_branches.extend(config['JA_RULE_EXTRA_BRANCHES'])
for branch in ja_rule_branches:
  builder_branches['ja_rule'][branch] = []
c['change_source'].append(
    GitPoller(BuildRepoURL(config['JA_RULE_REPO']),
              workdir='gitpoller-workdir',
              branches=ja_rule_branches,
              pollinterval=300,
              project='ja-rule')
)

################################################################################
# BUILDSLAVES

c['slavePortnum'] = config['SLAVE_PORT']

# The 'slaves' list defines the set of recognized buildslaves. Each element is
# a BuildSlave object, specifying a unique slave name and password.
c['slaves'] = [BuildSlave(s.name(), s.password()) for s in slaves.GetSlaves()]

################################################################################
# BUILDERS

# The 'builders' list defines the Builders, which tell Buildbot how to perform
# a build: what steps, and which slaves can execute them.  Note that any
# particular build will only take place on one slave.

# OLA factories

# The main factories, to do fetch via git, autoreconf, configure, make & make
# check, with or without Ja Rule
ola_default_factory = BuildFactory()
ola_no_ja_rule_factory = BuildFactory()
# check out the source
git_step = Git(repourl=BuildRepoURL(config['OLA_REPO']))
ola_default_factory.addStep(git_step)
ola_no_ja_rule_factory.addStep(git_step)
autoreconf_step = ShellCommand(
    command=["autoreconf", "-i"],
    name="autoreconf",
    haltOnFailure=True,
    flunkOnFailure=True,
)
ola_default_factory.addStep(autoreconf_step)
ola_no_ja_rule_factory.addStep(autoreconf_step)
# Should be able to switch to these with newer versions of buildbot
# ola_default_factory.addStep(Configure(command=["./configure",
#                                                "--enable-e133",
#                                                "--enable-rdm-tests",
#                                                "--enable-ja-rule"]))
ola_default_factory.addStep(ShellCommand(
    command=["./configure",
             "--enable-e133", "--enable-rdm-tests", "--enable-ja-rule"],
    name="configure",
    haltOnFailure=True,
    flunkOnFailure=True,
))
ola_no_ja_rule_factory.addStep(ShellCommand(
    command=["./configure", "--enable-e133", "--enable-rdm-tests"],
    name="configure",
    haltOnFailure=True,
    flunkOnFailure=True,
))
compile_step = Compile(
    name='make',
    description='building',
    descriptionDone='build',
    command=['make'],
)
ola_default_factory.addStep(compile_step)
ola_no_ja_rule_factory.addStep(compile_step)
test_step = Compile(
    name='make check',
    description='testing',
    descriptionDone='test',
    command=['make', 'check'],
    logfiles=config['OLA_LOGFILES'],
    lazylogfiles=True,
)
ola_default_factory.addStep(test_step)
ola_no_ja_rule_factory.addStep(test_step)

# OLA cpplint factory
if 'OLA_CPP_LINT_ARGS' in config:
  ola_cpp_lint_command = (
      'cpplint.py %s $(find ./ -name "*.h" -or -name "*.cpp" | xargs)' %
      config['OLA_CPP_LINT_ARGS'])
  ola_cpp_lint_factory = BuildFactory()
  ola_cpp_lint_factory.addStep(Git(repourl=BuildRepoURL(config['OLA_REPO'])))
  ola_cpp_lint_factory.addStep(ShellCommand(
     command=ola_cpp_lint_command,
     haltOnFailure=False,
     flunkOnFailure=False,
     warnOnWarnings=True,
     warnOnFailure=True,
     description='linting',
     descriptionDone='lint',
  ))

# OLA jslint factory
if 'OLA_JS_LINT_ARGS' in config:
  ola_js_lint_command = (
      'gjslint %s -r javascript/ola/' %
      config['OLA_JS_LINT_ARGS'])
  ola_js_lint_factory = BuildFactory()
  ola_js_lint_factory.addStep(Git(repourl=BuildRepoURL(config['OLA_REPO'])))
  ola_js_lint_factory.addStep(ShellCommand(
      command=ola_js_lint_command,
      haltOnFailure=False,
      flunkOnFailure=False,
      warnOnWarnings=True,
      warnOnFailure=True,
      description='linting',
      descriptionDone='lint',
  ))

# OLA heap check factory
ola_hc_factory = BuildFactory()
ola_hc_factory.addStep(Git(repourl=BuildRepoURL(config['OLA_REPO'])))
ola_hc_factory.addStep(ShellCommand(command=["autoreconf", "-i"],
                                    name="autoreconf"))
ola_hc_factory.addStep(Configure(
    command=["./configure", "--enable-e133", "--enable-ja-rule",
             "--enable-tcmalloc"],
    name="configure",
))
ola_hc_factory.addStep(Compile(
    name='make',
    description='building',
    descriptionDone='build',
    command=['make']))
ola_hc_factory.addStep(Compile(
    name='make check',
    description='testing',
    descriptionDone='test',
    command=['make', 'check'],
    env={'HEAPCHECK': 'normal',
         'PPROF_PATH': '/usr/bin/google-pprof'},
    logfiles=config['OLA_LOGFILES'],
    lazylogfiles=True,
))

# OLA conf help factory
ola_conf_factory = BuildFactory()
ola_conf_factory.addStep(Git(repourl=BuildRepoURL(config['OLA_REPO'])))
ola_conf_factory.addStep(ShellCommand(
    workdir="build/plugins",
    command=["./generate-html.sh", "/opt/www/docs.openlighting.org/ola/conf/"],
    name="generate"))

# OLA doxygen doc factory
ola_doc_factory = BuildFactory()
ola_doc_factory.addStep(Git(repourl=BuildRepoURL(config['OLA_REPO'])))
ola_doc_factory.addStep(ShellCommand(command=["autoreconf", "-i"],
                                     name="autoreconf"))
# Should be able to switch to these with newer versions of buildbot
# ola_doc_factory.addStep(Configure(command=["./configure",
#                                            "--disable-doxygen-version"]))
ola_doc_factory.addStep(ShellCommand(
    command=["./configure", "--disable-doxygen-version"],
    name="configure",
))
ola_doc_factory.addStep(Compile(
    name='make builtfiles',
    description='generating built files',
    descriptionDone='generated built files',
    command=['make', 'builtfiles']))
ola_doc_factory.addStep(Compile(
    name='make doxygen-doc',
    description='documenting',
    descriptionDone='documented',
    flunkOnWarnings=True,
    command=['make', 'doxygen-doc']))
ola_doc_factory.addStep(ShellCommand(
    workdir="build",
    command=["./doxygen/copy-doc.sh",
             "/opt/www/docs.openlighting.org/ola/doc/latest/"],
    name="copy"))

# OLA man page factory
ola_man_factory = BuildFactory()
ola_man_factory.addStep(Git(repourl=BuildRepoURL(config['OLA_REPO'])))
ola_man_factory.addStep(ShellCommand(
    workdir="build/man",
    command=["./generate-html.sh", "/opt/www/docs.openlighting.org/ola/man/"],
    name="generate"))

# Ja-Rule factories
# Ja-Rule doxygen doc factory
ja_rule_doc_factory = BuildFactory()
ja_rule_doc_factory.addStep(Git(repourl=BuildRepoURL(config['JA_RULE_REPO'])))
ja_rule_doc_factory.addStep(ShellCommand(command=["./install-gmock.sh"],
                                         name="install gmock"))
ja_rule_doc_factory.addStep(ShellCommand(command=["autoreconf", "-i"],
                                         name="autoreconf"))
# Should be able to switch to these with newer versions of buildbot
# ja_rule_doc_factory.addStep(Configure(command=["./configure",
#                                                "--disable-doxygen-version"]))
ja_rule_doc_factory.addStep(ShellCommand(
    command=["./configure", "--disable-doxygen-version", "--without-ola"],
    name="configure",
))
# ja_rule_doc_factory.addStep(Compile(
#     name='make builtfiles',
#     description='generating built files',
#     descriptionDone='generated built files',
#     command=['make', 'builtfiles']))
ja_rule_doc_factory.addStep(Compile(
    name='make doxygen-doc',
    description='documenting',
    descriptionDone='documented',
    flunkOnWarnings=True,
    command=['make', 'doxygen-doc']))
ja_rule_doc_factory.addStep(ShellCommand(
    command=["./doxygen/copy-doc.sh",
             "/opt/www/docs.openlighting.org/ole/doc/latest/"],
    name="copy"))

# Ja-Rule doxygen manual factory
ja_rule_manual_factory = BuildFactory()
ja_rule_manual_factory.addStep(Git(
    repourl=BuildRepoURL(config['JA_RULE_REPO'])
))
ja_rule_manual_factory.addStep(ShellCommand(command=["./install-gmock.sh"],
                                            name="install gmock"))
ja_rule_manual_factory.addStep(ShellCommand(command=["autoreconf", "-i"],
                                            name="autoreconf"))
# Should be able to switch to these with newer versions of buildbot
# ja_rule_manual_factory.addStep(Configure(
#     command=["./configure", "--disable-doxygen-version", "--without-ola"]))
ja_rule_manual_factory.addStep(ShellCommand(
    command=["./configure", "--disable-doxygen-version", "--without-ola"],
    name="configure",
))
# ja_rule_manual_factory.addStep(Compile(
#     name='make builtfiles',
#     description='generating built files',
#     descriptionDone='generated built files',
#     command=['make', 'builtfiles']))
ja_rule_manual_factory.addStep(Compile(
    workdir="build/user_manual",
    name='doxygen',
    description='documenting',
    descriptionDone='documented',
    flunkOnWarnings=True,
    command=['doxygen']))
ja_rule_manual_factory.addStep(ShellCommand(
    workdir="build/user_manual",
    command=["./copy-manual.sh",
             "/opt/www/docs.openlighting.org/ole/manual/latest/"],
    name="copy"))

c['builders'] = []

# OLA builders
# Create a make, make check builder for each slave, as well as various other
# checks
# TODO: We should probably just create a builder for each (platform, arch) and
# add all the slaves instead. No point running a build more than once in the
# same environment.
# PN: Although it depends if we want to be able to distinguish a build failing
# due to a bug related to openslp say, compared to the arch/OS in general
for branch_name in builder_branches['ola'].keys():
  builder_base_name = ("ola" if branch_name is config['OLA_TRUNK_NAME']
                       else ("ola-%s" % branch_name))

  for slave in slaves.GetSlaves(config_helper.OlaSlave, config_helper.HasBuild):
    builder_name = ("buildcheck-%s-%s" % (builder_base_name, slave.name()))
    builder_branches['ola'][branch_name].append(builder_name)
    c['builders'].append(
        BuilderConfig(name=builder_name,
                      slavenames=[slave.name()],
                      factory=(ola_default_factory if slave.has_ola_ja_rule_deps
                               else ola_no_ja_rule_factory),
                      # Slow slaves will merge requests, to try and reduce their
                      # workload
                      mergeRequests=slave.is_slow,
                      tags=['ola', ("ola-%s" % branch_name)]))

  ola_cpp_lint_slaves = slaves.GetSlaves(config_helper.OlaSlave,
                                         config_helper.HasCPPLintFilter)
  if ('OLA_CPP_LINT_ARGS' in config) and ola_cpp_lint_slaves:
    builder_name = ("cpplint-%s" % builder_base_name)
    builder_branches['ola'][branch_name].append(builder_name)
    c['builders'].append(
        BuilderConfig(name=builder_name,
                      slavenames=[s.name() for s in ola_cpp_lint_slaves],
                      factory=ola_cpp_lint_factory,
                      tags=['ola', ("ola-%s" % branch_name)]))

  ola_js_lint_slaves = slaves.GetSlaves(config_helper.OlaSlave,
                                        config_helper.HasJSLintFilter)
  if ('OLA_JS_LINT_ARGS' in config) and ola_js_lint_slaves:
    builder_name = ("jslint-%s" % builder_base_name)
    builder_branches['ola'][branch_name].append(builder_name)
    c['builders'].append(
        BuilderConfig(name=builder_name,
                      slavenames=[s.name() for s in ola_js_lint_slaves],
                      factory=ola_js_lint_factory,
                      tags=['ola', ("ola-%s" % branch_name)]))

  ola_tcmalloc_slaves = slaves.GetSlaves(config_helper.OlaSlave,
                                         config_helper.HasTCMalloc)
  if ola_tcmalloc_slaves:
    builder_name = ("leakchecker-%s" % builder_base_name)
    builder_branches['ola'][branch_name].append(builder_name)
    c['builders'].append(
        BuilderConfig(name=builder_name,
                      slavenames=[s.name() for s in ola_tcmalloc_slaves],
                      factory=ola_hc_factory,
                      tags=['ola', ("ola-%s" % branch_name)]))

# Trunk only builders, these generally generate output to the website, so we
# only run them against the bleeding edge trunk branch
ola_conf_slaves = slaves.GetSlaves(config_helper.OlaSlave,
                                   config_helper.GenerateConf)
if ola_conf_slaves:
  builder_branches['ola'][config['OLA_TRUNK_NAME']].append(
      "conf-page-generator-ola")
  c['builders'].append(
      BuilderConfig(name="conf-page-generator-ola",
                    slavenames=[s.name() for s in ola_conf_slaves],
                    factory=ola_conf_factory,
                    tags=['ola', ("ola-%s" % config['OLA_TRUNK_NAME'])]))

ola_doc_slaves = slaves.GetSlaves(config_helper.OlaSlave,
                                  config_helper.GenerateDoc)
if ola_doc_slaves:
  builder_branches['ola'][config['OLA_TRUNK_NAME']].append(
      "doxygen-doc-generator-ola")
  c['builders'].append(
      BuilderConfig(name="doxygen-doc-generator-ola",
                    slavenames=[s.name() for s in ola_doc_slaves],
                    factory=ola_doc_factory,
                    tags=['ola', ("ola-%s" % config['OLA_TRUNK_NAME'])]))

ola_man_slaves = slaves.GetSlaves(config_helper.OlaSlave,
                                  config_helper.GenerateMan)
if ola_man_slaves:
  builder_branches['ola'][config['OLA_TRUNK_NAME']].append(
      "man-page-generator-ola")
  c['builders'].append(
      BuilderConfig(name="man-page-generator-ola",
                    slavenames=[s.name() for s in ola_man_slaves],
                    factory=ola_man_factory,
                    tags=['ola', ("ola-%s" % config['OLA_TRUNK_NAME'])]))

# Ja-Rule builders
# Trunk only builders, these generally generate output to the website, so we
# only run them against the bleeding edge branch

ja_rule_doc_slaves = slaves.GetSlaves(config_helper.JaRuleSlave,
                                      config_helper.GenerateDoc)
if ja_rule_doc_slaves:
  builder_branches['ja_rule'][config['JA_RULE_TRUNK_NAME']].append(
      "doxygen-doc-generator-ja-rule")
  c['builders'].append(
      BuilderConfig(name="doxygen-doc-generator-ja-rule",
                    slavenames=[s.name() for s in ja_rule_doc_slaves],
                    factory=ja_rule_doc_factory,
                    tags=['ja-rule', ("ja-rule-%s" %
                                      config['JA_RULE_TRUNK_NAME'])]))
  builder_branches['ja_rule'][config['JA_RULE_TRUNK_NAME']].append(
      "doxygen-manual-generator-ja-rule")
  c['builders'].append(
      BuilderConfig(name="doxygen-manual-generator-ja-rule",
                    slavenames=[s.name() for s in ja_rule_doc_slaves],
                    factory=ja_rule_manual_factory,
                    tags=['ja-rule', ("ja-rule-%s" %
                                      config['JA_RULE_TRUNK_NAME'])]))

################################################################################
# SCHEDULERS

# Configure the Schedulers, which decide how to react to incoming changes.  In
# this case, just kick off all the builds for that project against the
# particular branch

# Add the simple schedulers
c['schedulers'] = [
    Try_Userpass(
        name='ola-try',
        builderNames=[b.name for b in c['builders'] if ('ola' in b.name)],
        port=5555,
        userpass=[('sampleuser', 'samplepass')],
    ),
    # Try_Userpass(
    #     name='ja-rule-try',
    #     builderNames=[b.name for b in c['builders'] if ('ja-rule' in b.name)],
    #     port=5555,
    #     userpass=[('sampleuser', 'samplepass')],
    # ),
]

for branch_name in builder_branches['ola'].keys():
  scheduler_base_name = ("ola" if branch_name is config['OLA_TRUNK_NAME']
                         else ("ola-%s" % branch_name))
  c['schedulers'].append(
      SingleBranchScheduler(
          name=("%s-all" % scheduler_base_name),
          change_filter=filter.ChangeFilter(branch=branch_name,
                                            project='ola'),
          treeStableTimer=None,
          builderNames=builder_branches['ola'][branch_name],
      )
  )
  c['schedulers'].append(
      ForceScheduler(
          name=("%s-force" % scheduler_base_name),
          builderNames=builder_branches['ola'][branch_name],
          project=FixedParameter(name="project", default="ola"),
          repository=FixedParameter(
              name="repository",
              default=BuildRepoURL(config['OLA_REPO'])),
          branch=FixedParameter(name="branch", default=branch_name),
      )
  )

for branch_name in builder_branches['ja_rule'].keys():
  scheduler_base_name = ("ja-rule" if (branch_name is
                                       config['JA_RULE_TRUNK_NAME'])
                         else ("ja-rule-%s" % branch_name))
  c['schedulers'].append(
      SingleBranchScheduler(
          name=("%s-all" % scheduler_base_name),
          change_filter=filter.ChangeFilter(branch=branch_name,
                                            project='ja-rule'),
          treeStableTimer=None,
          builderNames=builder_branches['ja_rule'][branch_name],
      )
  )
  c['schedulers'].append(
      ForceScheduler(
          name=("%s-force" % scheduler_base_name),
          builderNames=builder_branches['ja_rule'][branch_name],
          project=FixedParameter(name="project", default="ja-rule"),
          repository=FixedParameter(
              name="repository",
              default=BuildRepoURL(config['JA_RULE_REPO'])),
          branch=FixedParameter(name="branch", default=branch_name),
      )
  )


################################################################################
# STATUS TARGETS

# 'status' is a list of Status Targets. The results of each build will be
# pushed to these targets. buildbot/status/*.py has a variety to choose from,
# including web pages, email senders, and IRC bots.

c['status'] = []

authz_cfg = authz.Authz(
    # change any of these to True to enable; see the manual for more
    # options
    auth=auth.BasicAuth([("ola", "ola"), ("ja-rule", "ja-rule")]),
    gracefulShutdown=False,
    forceBuild='auth',  # use this to test your slave once it is set up
    forceAllBuilds=False,
    pingBuilder=False,
    stopBuild='auth',
    stopAllBuilds=False,
    cancelPendingBuild='auth',
)
c['status'].append(
    html.WebStatus(
        http_port=8010,
        authz=authz_cfg))

# Reverting to one IRC bot, as buildbot doesn't seem to cope with two properly
# Quiet bot, temporarily in test channel, as it seems to be too chatty
# if (('IRCSERVER' in config) and ('IRCBOTBASENAME' in config) and
#     ('IRCBOTCHANNEL' in config)):
#   c['status'].append(words.IRC(host=config['IRCSERVER'],
#                                nick=("%s%s" % (config['IRCBOTBASENAME'],
#                                                "-qu")),
#                                allowForce=True,
#                                channels=[config['IRCCHANNEL']],
#                                notify_events={
#                                  'started': 0,
#                                  'finished': 0,
#                                  'success': 0,
#                                  'failure': 0,
#                                  'exception': 0,
#                                  'successToFailure': 1,
#                                  'failureToSuccess': 1,
#                                }))

# IRC Bot, very chatty, in a configurable channel, probably it's own
if (('IRCSERVER' in config) and ('IRCBOTBASENAME' in config) and
    ('IRCBOTCHANNEL' in config)):
  c['status'].append(words.IRC(
    host=config['IRCSERVER'],
    # If we just have one bot, we only need one name
    # nick=("%s%s" % (config['IRCBOTBASENAME'],
    #                 "-ch")),
    nick=config['IRCBOTBASENAME'],
    allowForce=True,
    channels=[config['IRCBOTCHANNEL']],
    notify_events={
      'started': 1,
      'finished': 1,
      'success': 1,
      'failure': 1,
      'exception': 1,
      'successToFailure': 1,
      'failureToSuccess': 1,
    })
  )

################################################################################
# DB URL

c['db'] = {
    'db_url': "sqlite:///state.sqlite",
}