cmsBuild

#!/usr/bin/python -u
#
# Driver script for building rpms from a configuration specified in CMSDIST.
#

from os.path import abspath, join, exists, isdir, basename, dirname
from os import popen, getenv, symlink, listdir, readlink, unlink, getpid
from os import chdir, getcwd, environ, walk, sep, kill
from getpass import getuser
from tempfile import mkdtemp, mkstemp, NamedTemporaryFile
from commands import getstatusoutput
from urllib2 import urlopen, URLError
from time import strftime
import sys
import copy
from ConfigParser import ConfigParser
from ConfigParser import InterpolationMissingOptionError
import re, os
import traceback
from optparse import OptionParser, OptionGroup
import fnmatch
from shutil import rmtree
import urllib2
from glob import glob
import itertools
import pickle

logLevel = 10 
NORMAL=10
DEBUG=20
TRACE=30
DEFAULT_CVS_SERVER=":pserver:anonymous@cmscvs.cern.ch:2401/local/reps/CMSSW"
DEFAULT_CVS_PASSWORD="AA_:yZZ3e"
DEFAULT_SPEC_REPOSITORY_MODULE="CMSDIST"

removeInitialSpace = re.compile ("^[ ]*", re.M)
cmsBuildFilename = abspath(__file__)

def fatal(message):
  print "ERROR: " + message
  sys.exit(1)

def die(message):
  fatal(message)

# Minimal string sanitization.
def sanitize(s):
  return re.sub("[^a-zA-Z_0-9*./-]", "", s)

def log (message, level=0):
    """ Asynchronous printouts method. Level should be NORMAL when a message 
is intended to be seen by the user, DEBUG, when it's intended to be seen only 
by the developer, and TRACE when it's a message that is describing state 
of the action scheduler/worker during the build.
    """
    if callable (message):
        message = message ()
    if level <= logLevel:
        message = removeInitialSpace.sub ("", message)
        print message

def setLogLevel (options):
    global logLevel
    if options.trace:
        logLevel = TRACE
    elif options.debug:
        logLevel = DEBUG
    else:
        logLevel = NORMAL

#Check of online arch
def isOnline(arch):
    if 'onl' in arch: return True
    return False

def isDefaultRevision(rev):
    if rev == '1' or rev.startswith('1.'): return True
    return False

# We have our own version rather than using the one from os
# because the latter does not work seem to be thread safe.
def makedirs(path):
  returncode, out = getstatusoutput("mkdir -p %s" % (path,))
  if returncode != 0:
    raise OSError("makedirs() failed (return: %s):\n%s" % (returncode, out)) 

# A recursive globbing helper.
def recursive_glob(treeroot, pattern):
  results = []
  for base, dirs, files in walk(treeroot):
    goodfiles = fnmatch.filter(files, pattern) + fnmatch.filter(dirs, pattern)
    results.extend(join(base, f) for f in goodfiles)
  return results

rpm_db_cache = {}
    
def downloadUrllib2(source, destDir, options):
    try:
      dest = "/".join([destDir.rstrip("/"), basename(source)])
      req = urllib2.Request(source, headers={"Cache-Control": "no-cache"})
      s = urllib2.urlopen(req)
      f = file(dest+".tmp","w")
      # Read in blocks to avoid using too much memory.
      block_sz = 8192*16
      while True:
        buffer = s.read(block_sz)
        if not buffer:
          break
        f.write(buffer)
      f.close()
      os.rename(dest+".tmp",dest)
    except URLError, e:
      if not "SOURCES/cache" in source:
        log("Error while downloading %s: %s" % (source, e))
      return False
    except Exception, e:
      print source, destDir
      log("Error while downloading %s: %s" % (source, e))
      return False
    return True

def parseUrl (url, requestedKind=None, defaults={}, required=[]):
    match = re.match("([^+:]*)([^:]*)://([^?]*)(.*)", url)
    if not match:
      raise MalformedUrl (url)
    parts = match.groups()
    protocol, deliveryProtocol, server, arguments = match.groups()
    arguments = arguments.strip("?")
    # In case of urls of the kind:
    # git+https://some.web.git.repository.net
    # we consider "https" the actual protocol and
    # "git" merely the request kind.
    if requestedKind and not protocol == requestedKind:
      raise MalformedUrl(url)
    if deliveryProtocol:
      protocol = deliveryProtocol.strip("+")
    arguments.replace ("&amp;", "&")
    args = defaults.items ()
    parsedArgs = re.split ("&", arguments)
    parsedArgs = [ x.split ("=") for x in parsedArgs ]
    parsedArgs = [(len (x) != 2 and [x[0], True]) or x for x in parsedArgs]
    args.extend (parsedArgs)
    argsDict = dict(args)
    missingArgs = [arg for arg in required if arg not in argsDict]
    if missingArgs:
      raise MalformedUrl (url, missingArgs)
    return protocol, server, argsDict

def parseTcUrl (url):
    scheme, server, args = parseUrl (url, requestedKind="cmstc",
                                     defaults={"cvsroot": DEFAULT_CVS_SERVER,
                                               "passwd": DEFAULT_CVS_PASSWORD},
                                     required=["tag", "output"])
    return scheme, server, args

def parseCvsUrl (url):
    scheme, cvsroot, args = parseUrl (url, requestedKind="cvs",
                                      defaults={"strategy": "export",
                                                "tag": "HEAD",
                                                "passwd": DEFAULT_CVS_PASSWORD},
                                      required=["module", "output"])
    if not cvsroot:
        cvsroot = DEFAULT_CVS_SERVER
    cvsroot = cvsroot.replace (":/", ":2401/")
    args["tag"] = re.sub (re.compile ("^-r"), "", args["tag"])
    if "export" not in args:
        args["export"] = args["module"]
    args["cvsroot"] = cvsroot
    return (scheme, cvsroot, args)

def parseSvnUrl (url):
  scheme, root, args = parseUrl(url, requestedKind="svn",
                                defaults={"strategy": "export",
      			            "revision": "HEAD"},
                                required=["module", "output"])
  if "export" not in args:
    args["export"] = args["module"]
  if "scheme" in args:
    scheme=args["scheme"]
  if not scheme in ["svn", "http", "https", "file"] and (not scheme.startswith('svn+')):
    scheme = "svn+" + scheme
  args["svnroot"] = "%(scheme)s://%(root)s" % {"scheme": scheme, "root": root}
  return (scheme, root, args)

def parseGitUrl(url):
  protocol, gitroot, args = parseUrl(url, requestedKind="git",
                                     defaults={"obj": "master/HEAD"})
  parts = args["obj"].rsplit("/", 1)                                                                                                                                                                                                                                                              
  if len(parts) != 2:
    parts += ["HEAD"]
  args["branch"], args["tag"] = parts

  if not "export" in args:
    args["export"] = basename(re.sub("\.git$", "", re.sub("[?].*", "",gitroot)))
    if args["tag"] != "HEAD":
      args["export"] += args["tag"]
    else:
      args["export"] += args["branch"]

  if not "output" in args:
    args["output"] = args["export"] + ".tar.gz"
    args ["gitroot"] = gitroot
  if not "filter" in args:
    args["filter"] = "*"
  args["filter"] = sanitize(args["filter"]);
  return protocol, gitroot, args

def createTempDir (workDir, subDir):
    tempdir = join (workDir, subDir) 
    if not exists (tempdir):
        makedirs (tempdir)
    tempdir = mkdtemp (dir=tempdir)
    return tempdir

def executeWithErrorCheck (command, errorMessage):
    log (command, DEBUG)
    error, output = getstatusoutput (command)
    if error:
        log (errorMessage + ":")
        log ("")
        log (command)
        log ("")
        log ("resulted in:")
        log (output)
        return False
    log (output, DEBUG)
    return True

def packCheckout (tempdir, dest, *exports):
    """ Use this helper method when download protocol is like cvs/svn/git
        where the code is checked out in a temporary directory and then tarred 
        up.
    """
    export = " ".join(['"%s"' % x for x in exports])
    packCommand ="""cd %(tempdir)s; tar -zcf "%(dest)s" %(export)s """
    packCommand = packCommand % locals ()
    errorMessage = "Error while creating a tar archive for checked out area"
    return executeWithErrorCheck (packCommand, errorMessage)


def downloadSvn (source, dest, options):
    scheme, svnroot, args = parseSvnUrl (source)
    tempdir = createTempDir (options.workDir, options.tempDirPrefix)
    exportDir = join (tempdir, "checkout", args["export"])
    if not exists (exportDir):
        makedirs (exportDir)
    args["dest"] = join (dest, args["output"].lstrip("/"))
    args["tempdir"] = tempdir
    args["exportdir"] = exportDir.rsplit("/", 1)[1]
    args["exportpath"] = exportDir.rsplit("/", 1)[0]
    if not args["revision"].isdigit():
      args["revision"] = '"' + args["revision"] + '"' 
    command = """cd %(exportpath)s ; svn %(strategy)s --force --non-interactive --trust-server-cert -r%(revision)s "%(svnroot)s" "%(exportdir)s" """
    command = command % args
    message = "Error while downloading files from subversion repository"
    ok = executeWithErrorCheck (command, message)
    return ok and packCheckout (args["exportpath"], args["dest"], args["export"])

def downloadCvs (source, dest, options):
    protocol, cvsroot, args = parseCvsUrl (source)
    tempdir = createTempDir (options.workDir, options.tempDirPrefix)
    pserverUrlRe = re.compile (":pserver:.*")
    isPserver = pserverUrlRe.match (cvsroot)
    cvspassFilename = None
    if args.has_key ("passwd") and isPserver:
        cvspassFilename = join (tempdir, "cvspass")
        f=open (join (tempdir, "cvspass"), "w")
        f.write ("/1 %(cvsroot)s %(passwd)s\n" % args)
        f.close ()
    exportDir = join (tempdir, "checkout")
    if not exists (exportDir):
        makedirs (exportDir)
    args["dest"] = join (dest, args["output"].lstrip("/"))
    args["tempdir"] = tempdir
    args["exportdir"] = exportDir
    args["passexport"] = (cvspassFilename and "CVS_PASSFILE=%s" % cvspassFilename) or ""
    command ="""cd %(exportdir)s ; %(passexport)s cvs -z6 -Q -d"%(cvsroot)s" %(strategy)s -d"%(export)s" -r"%(tag)s" "%(module)s" """
    command = command % args
    message = "Error while executing a %(strategy)s from CVS:" % args
    ok = executeWithErrorCheck (command, message)
    return ok and packCheckout (exportDir, args["dest"], args["export"])

def downloadTc (source, dest, options):
  protocol, tagsSource, args = parseTcUrl (source)
  tempdir = createTempDir (options.workDir, options.tempDirPrefix)
  args["tempdir"] = tempdir
  args["dest"] = join(dest, args["output"].lstrip("/"))
  args["pmLocation"] = abspath(join (dirname(cmsBuildFilename), 'cmspm'))
  if "extratags" in args:
    args["extratags"] = "--additional-tags " + args["extratags"]
  else:
    args["extratags"] = ""

  if 'baserel' not in args:
    command="""cd %(tempdir)s && %(pmLocation)s corel %(tag)s """
  else:
    command='. '+options.workDir+"""/cmsset_default.sh && cd %(tempdir)s && %(pmLocation)s frombase %(tag)s %(baserel)s %(baserelver)s """
  command +=""" %(extratags)s -e -o src/PackageList.cmssw"""
  command=command % args
  log("Downloading %(tag)s from cmsTC." % args, DEBUG)
  message = "Error while downloading sources using tag collector information."
  ok = executeWithErrorCheck (command, message)
  dirs = ["src"]
  if exists("/".join([args["tempdir"], "poison"])): dirs.append("poison")
  return ok and packCheckout (args["tempdir"], args["dest"], *dirs)

# Download a files from a git url.  We do not clone the remote reposiotory, but
# we simply pull the branch we are interested in and then we drop all the git
# information while creating a tarball.  The syntax to define a repository is
# the following:
#
# git:/local/repository?obj=BRANCH/TAG
# git://remote-repository?obj=BRANCH/TAG
# git+https://remote-repository-over-http/foo.git?obj=BRANCH/TAG
#
# If "obj" does not contain a "/", it's value will be considered a branch and TAG will be "HEAD".
# If "obj" is not specified, it will be "master/HEAD" by default.
# By default export is the <basename of the url without ".git">-TAG unless it is HEAD,
# in which case it will be  <basename of the url without .git>-BRANCH.
# One can specify an additional parameter
#
#     filter=<some-path>
#
# which will be used to pack only a subset of the checkout.
def downloadGit(source, dest, options):
  protocol, gitroot, args = parseGitUrl(source)
  tempdir = createTempDir(options.workDir, options.tempDirPrefix)

  exportpath = join(tempdir, args["export"])
  if protocol:
    protocol += "://"
  if not protocol and not gitroot.endswith(".git"):
    gitroot = join(gitroot, ".git")

  dest = join(dest, args["output"].lstrip ("/"))
  args.update({"protocol": protocol, "tempdir": tempdir,
               "gitroot": gitroot, "dest": dest,
               "exportpath": exportpath})
  makedirs(exportpath)
  command = format("cd %(exportpath)s &&"
                   "git init &&"
                   "git pull --tags %(protocol)s%(gitroot)s refs/heads/%(branch)s &&"
                   "git reset --hard %(tag)s &&"
                   "find . ! -path '%(filter)s' -delete &&"
                   "rm -rf .git .gitattributes .gitignore", **args)
  error, output = getstatusoutput(command % args)
  if error:
    log("Error while downloading sources from %s using git.\n\n"
        "%s\n\n"
        "resulted in:\n%s" % (gitroot, command % args, output))
    return False
  return packCheckout(args["tempdir"], args["dest"], args["export"])

downloadHandlers = {'cvs': downloadCvs,
                    'cmstc': downloadTc,
                    'http': downloadUrllib2,
                    'https': downloadUrllib2,
                    'ftp': downloadUrllib2,
                    'ftps': downloadUrllib2,
                    'git': downloadGit,
                    'svn': downloadSvn}

    
def getUrlChecksum (s):
    m = md5adder (s)
    return m.hexdigest ()

def isProcessRunning(pid):
  running = False
  try:
    os.kill(pid, 0)
    running = True
  except:
    pass
  return running

class cmsLock (object):
  def __init__ (self, dirname):
    self.piddir  = join(dirname,".cmsLock")
    self.pidfile = join(self.piddir,"pid")
    self.pid     = str(getpid())
    self._hasLock = False
    self._hasLock = self._get()

  def __del__(self):
    self._release ()    

  def __nonzero__(self):
    return self._hasLock

  def _release (self, force=False):
    if (self._hasLock or force):
      try:
        if exists (self.piddir): getstatusoutput ("rm -rf %s" % self.piddir)
      except:
        pass
    self._hasLock = False

  def _get(self, count=0):
    if count >= 5: return False
    pid = self._readPid()
    if pid:
      if pid == self.pid: return True
      if isProcessRunning(int(pid)): return False
    self._create()
    sleep(0.0001)
    return self._get(count+1)

  def _readPid(self):
    pid = None
    try:
      pid = open(self.pidfile).readlines()[0]
    except:
      pid = None
    return pid

  def _create(self):
    self._release(True)
    try:
      makedirs(self.piddir)
      lock = open (self.pidfile, 'w')
      lock.write(self.pid)
      lock.close()
    except:
      pass

# Helper class which contains only the options that are
# relevant for download.
class DownloadOptions(object):
  def __init__ (self, options):
    self.workDir = options.workDir 
    self.tempDirPrefix = options.tempDirPrefix
    self.cmsdist = options.cmsdist 

def download (source, dest, options):
    # Syntactic sugar for git:/some/path to be equal to git+:///some/path
    if source.startswith("git:") and not source.startswith("git://"):
      source = "git+://" + source[4:]
    # Syntactic sugar to allow the following urls for tag collector:
    #
    # cmstc:[base.]release[.tagset[.tagset[...]]]/src.tar.gz
    #
    # in place of:
    #
    # cmstc://?tag=release&baserel=base&extratag=tagset1,tagset2,..&module=CMSSW&export=src&output=/src.tar.gz
    if source.startswith("cmstc:") and not source.startswith("cmstc://"):
      url = source.split(":", 1)[1]
      desc, output = url.rsplit("/", 1)
      parts = desc.split(".")
      releases = [x for x in parts if not x.isdigit()]
      extratags = [x for x in parts if x.isdigit()]
      if extratags:
        extratags = "&extratags=" + ",".join(extratags)
      if len(releases) == 1:
        baserel=""
        release="tag="+ releases[0]
      elif len(releases) == 2:
        baserel="&baserel=" + releases[0]
        release=releases[1]
      else:
        raise MalformedUrl(source)
      source = "cmstc://?%s%s%s&module=CMSSW&export=src&output=/%s" % (release,baserel,extratags,output)
      
    cacheDir=abspath (join (options.workDir, "SOURCES/cache")) 
    urlTypeRe = re.compile ("([^:+]*)([^:]*)://.*")
    match = urlTypeRe.match (source)
    if not urlTypeRe.match (source):
        raise MalformedUrl (source)
    downloadHandler = downloadHandlers[match.group (1)]
    checksum = getUrlChecksum (source)
    filename = source.rsplit("/", 1)[1]
    downloadDir = join (cacheDir, checksum)
    try:
      makedirs (downloadDir)
    except OSError, e:
      if not exists(downloadDir):
        raise downloadDir 

    realFile = join (downloadDir,filename)
    if exists (realFile) and not exists (join (dest, filename)):
        symlink (realFile, join (dest, filename))
        return True
    if exists (realFile):
        return True
    cachedFile = "%s/%s/SOURCES/cache/%s/%s" % (options.server.rstrip ("/"), options.repository, checksum, filename)
    downloadOptions = DownloadOptions(options)
    log ("Trying to fetch cached file: %s" % cachedFile, DEBUG)
    success = downloadHandlers["http"] (cachedFile, downloadDir, downloadOptions)
    if not success:
        success = downloadHandler (source, downloadDir, downloadOptions)
    if success:
        f=open (join (downloadDir, "url"), 'w')
        f.write ("%s\n" % source)
        f.close ()
        if exists (realFile) and not exists (join (dest, filename)):
            symlink (realFile, join (dest, filename))        
    return success

class MalformedUrl (Exception):
    def __init__ (self, url, missingParams=[]):
        if not missingParams:
            self.args = ["ERROR: The following url is malformed: %(url)s." % locals ()]
        else:
            self.args = ["ERROR: The following parameters are missing from url %(url)s: %(missingParams)s" % locals ()]

class MalformedSpec (Exception):
    pass

class RpmBuildFailed (Exception):
    def __init__ (self, package):
        self.args = ["Failed to build package %s." % package.name] 
        self.pkg = package

class UnexpectedFile (Exception):
    def __init__ (self, filename):
        self.args = ["""Unexpected file:\n %s\n
Please remove it and start again.""" % filename]

class RpmInstallFailed (Exception):
    def __init__ (self, package, why=""):
        self.args = ["Failed to install package %s. Reason:\n%s" % (package.name, why)]
        self.pkg = package
        self.why = why

class UnableToDownload (Exception):
    pass

class FileNotFound (Exception):
    def __init__ (self, filename):
        log (filename)
        self.filename = filename
    def __repr__ (self):
        log ("Unable to find file %s" % self.filename)

class NotCorrectlyBootstrapped (Exception):
    def __init__ (self, why):
        self.why = why

class UnknownCompiler (Exception):
    pass

# The initial id for a given tag. Change to 1 if you always want to have -cms.
INITIAL_ID = 0
    
def tagToId (tag, origTag):
    return int ((tag or 0) and (tag.replace (origTag, "") or 1))

def idToTag (tagId, origTag):
    if not tagId:
        return ""
    elif tagId == 1:
        return origTag
    else:
        return "%s%s" % (origTag, tagId)

# Parses the `### RPM <group> <package> <realversion>` header.
# Such a header defines the package name, group and real version
# for a given package.
# Notice that for a package which matches the compiler
# name we allow the overriding of its version with the one specified 
# in the `compilerVersion` settings.
# Notice also that the one can specify /bin/sh commands in backticks
# for the version, so that stuff like the date can be added to the
# version (useful for IB and alikes).
# FIXME: notice that for compatibility with old packages, which were
# specifying a -CMSXYZ tag by hand, we remove such a suffix from 
# the version. This is probably not needed anymore and can go.
def parseRPMLine (specLines, opts):
  findRpmRe = re.compile ("^### RPM[ ]*([^ ]*)\s*([^ ]*)\s*(.*)")
  for line in specLines:
    match = findRpmRe.match (line)
    if not match:
      continue
    results = [x.strip (" ") for x in match.groups ()]
    results[2] = re.sub ("-CMS.*","",results[2])
    group, name, version = results
    error, output = getstatusoutput("echo %s" % version)
    if error:
      raise MalformedSpec
    version = output.split("\n")[0]
    if opts.compilerVersion and name == opts.compilerName:
      version = opts.compilerVersion
    return (group, name, version)
  raise MalformedSpec

def parseNoCompilerLine(specLines):

    findNoCompilerRe = re.compile ("^## NOCOMPILER")
    for line in specLines:
        if findNoCompilerRe.match(line):
            return True
    return False

def parseBuildRequireToolfileLine(specLines):

    findBRToolfileRe = re.compile ("^## BUILDREQUIRE-TOOLFILE")
    for line in specLines:
        if findBRToolfileRe.match(line):
            return True
    return False

class PkgInfo (object):
    """ Minimal information about a package.
    """
    def __init__ (self, pkg):
        self.name = pkg.name
        self.realVersion = pkg.realVersion
        self.group = pkg.group
        self.checksum = pkg.checksum
        self.cmsplatf = pkg.cmsplatf

    def id (self):
        return "%(group)s+%(name)s+%(realVersion)s" % self.__dict__

def getPkgName (filename):
    return re.match ("(.*)-1-[1-9]+[.].*", basename(filename)).group (1)
  
def getPkgChecksumFile (officialRpmLocation, buildOptions=None):
    try:
        link = readlink (officialRpmLocation)
    except OSError:
        log ("""ERROR! File %(officialRpmLocation)s is not a link!?!? 
                Are you running in an old cmsBuild.sh/install.sh area?
                If so, please remove the old package by doing:

                rm -rf %(officialRpmLocation)s

                and try again.
                """ % locals ())
        sys.exit (1)
    if not exists (link):
        packageName = getPkgName (officialRpmLocation)
        cmsBuildExec = sys.argv[0]
        workdir = buildOptions.workDir and "--work-dir %s"% buildOptions.workDir or ""
        doNotBootstrap = buildOptions.bootstrap and "" or "--do-not-bootstrap"
        architecture = "--architecture %s" % buildOptions.architecture
        cmsdist = "--cmsdist %s" % buildOptions.cmsdist
        brokenLink = join(buildOptions.workDir, officialRpmLocation) 
        log ("""ERROR: File
        
                %(brokenLink)s 
        
                links to 
                
                %(link)s 
                
                but the latter does not exists. Please run:
                
                # %(cmsBuildExec)s %(cmsdist)s %(architecture)s %(workdir)s %(doNotBootstrap)s deprecate-local %(packageName)s
            
                to fix the problem and then try again.
            """ % locals ())
        sys.exit (1)
    checksum = re.match (".*/(.*)/.*/.*", link).groups ()[0]
    if not checksum:
        log ("""ERROR: malformed link found: %(link)s.""" % locals ())
        sys.exit (1)
    return checksum

class TagCacheAptImpl (object):
    """ Concrete implementation of the tag cache which relies on the
        apt repository information and locally found packages. Consistency
        is enforced by policy (only one person is allowed to upload packages 
        for a given tag) and by checking at upload time that packages are not
        already in the repository. 
    """
    def __init__ (self, options):
        self.options = options
    
    def update (self):
      if self.options.bootstrap:
        aptGetUpdateCommand = "apt-get update -o Acquire::http::No-Cache=true -o Acquire::http::Pipeline-Depth=0"
        error, output = getstatusoutput (aptGetUpdateCommand)
        if error:
          die("Error while executing apt-get update.\n%s" % output)
        aptCacheCommand = "apt-cache search SpecChecksum"
        error, output = getstatusoutput(aptCacheCommand)
        if error:
          die("Error while executing apt-cache search SpecCheckum.\n%s" % output)
        lines = [line for line in output.split("\n") if line]
        chksumRE = re.compile("\s+-\s+.*?SpecChecksum:")
        pairs = [chksumRE.sub(' ',line).split() for line in lines]
        try:
          self.cache = dict(pairs)
        except:
          die("Malformed apt-cache output.")
       
    def requestTag (self, pkgInfo, tag):
        """ requestTag returns a unique tag (i.e. the mnemonic suffix at the 
            end of the package name) based on what is found in the online 
            database and in the local build area.
            
            @a pkgInfo is the structure which holds the information about a 
               tag.
            @a tag the mnemonic tag to be used to keep track of different 
               package builds. E.g. "cms" or "ge"
            @return a unique tag (ie cmsXXX) which can be used for the package.
        """
        tags = {}
        if self.options.bootstrap:
            matchingPackages = [(n, self.cache[n]) for n in self.cache if pkgInfo.id() in n]
            for name, checksum in matchingPackages:
                tempTag = name.replace(pkgInfo.id (), "").strip ("-")
                tags[tempTag] = checksum
        for package in listdir (join ("RPMS", pkgInfo.cmsplatf)): 
            if pkgInfo.id() not in package:
                continue
            linkName = join ("RPMS", pkgInfo.cmsplatf, package)
            checksum = getPkgChecksumFile (linkName, self.options)
            tempTag = getPkgName(package).replace (pkgInfo.id (), "").strip ("-")
            tags[tempTag] = checksum
        for i in itertools.count(INITIAL_ID):
            finalTag = idToTag (i, tag)
            if finalTag not in tags:
                break
        return finalTag
    
    def getTag (self, pkg):
        """ getTag looks up for a tag associated to an md5 sum the following way.
        1) Look up in the cache apt-cache search results.
        2) Looks up in RPMS/cache/<checksum> to see if a package is already there.
        """
        output = ""
        if self.options.bootstrap:
          matchingPackages = [n for n in self.cache if self.cache[n]==pkg.checksum]
          if len(matchingPackages)>1:
            orderedPackages = []
            for n in matchingPackages:
              if exists("%s/RPMS/cache/" % self.options.workDir + "%(checksum)s/%(cmsplatf)s/" % pkg.__dict__ + n + "-%(rpmVersion)s-%(pkgRevision)s.%(cmsplatf)s.rpm" % pkg.__dict__):
                orderedPackages.insert(0,n)
              else:
                orderedPackages.append(n)
            matchingPackages = orderedPackages
          if len(matchingPackages):
            output = matchingPackages[0]
          else:
            return None
        else:
            try:
                assert (pkg.cmsplatf and pkg.cmsplatf != "%cmsplatf")
                assert (pkg.checksum and pkg.checksum != "%checksum" and pkg.checksum != "%{nil}")
                rpmCachePath = join (abspath ("RPMS"), 
                                     "cache/%(checksum)s/%(cmsplatf)s" % pkg.__dict__)
                if "%{" in rpmCachePath:
                    print "ERROR: you seem to be using an old rpm-preamble.file in your CMSDIST."
                    print "Please make sure you update it to revision 1.20 at least."
                    sys.exit(1)
                files = listdir (rpmCachePath % pkg.__dict__)
                if len (files) > 1 + len(pkg.subpackages):
                    log ("""ERROR: %s structure is hoosed. You might want to do:
                            cmsBuild remove %s""" % (rpmCachePath, 
                                                     getPkgName(files[0])))
                    sys.exit (1)
                elif not len (files):
                    return None
                output = getPkgName(files[0])
            except OSError:
                return None
        
        if not output:
            return None
        fullVersion = output.rsplit("+", 1)[1]
        if "-" not in fullVersion:
            return ""
        tag = fullVersion.rsplit("-", 1)[1]
        # Return tag if it is not part of realVersion
        if not pkg.realVersion.endswith("-"+tag):
            return tag
        return ""
    
    def packageChecksums (self, package):
        """ 
        """
        assert (False and "Not implemented for the time being.")

global tags_cache        
tags_cache = None

# FIXME: write a more valuable description
DEFAULT_SECTIONS = {"": """
""",
                    "%%description": """
No description
""",
                  "%prep": """
%%setup -n %n-%realversion
""",
                  "%build": """
%initenv
./configure --prefix=%i
make
""",
                "%install": """
%initenv
make install
""",
                "%pre": """
if [ X"$(id -u)" = X0 ]; then
    echo "*** CMS SOFTWARE INSTALLATION ABORTED ***"
    echo "CMS software cannot be installed as the super-user."
    echo "(We recommend reading a unix security guide)."
    exit 1
fi
""",
                "%post": """
if [ "X$CMS_INSTALL_PREFIX" = "X" ] ; then CMS_INSTALL_PREFIX=$RPM_INSTALL_PREFIX; export CMS_INSTALL_PREFIX; fi
%{relocateConfig}etc/profile.d/init.sh
%{relocateConfig}etc/profile.d/init.csh
""",
                "%preun": """
""",
                "%postun": """
if [ "X$CMS_INSTALL_PREFIX" = "X" ] ; then CMS_INSTALL_PREFIX=$RPM_INSTALL_PREFIX; export CMS_INSTALL_PREFIX; fi
""",
                "%files": """
%{i}/
%dir %{instroot}/
%dir %{instroot}/%{cmsplatf}/
%dir %{instroot}/%{cmsplatf}/%{pkgcategory}/
%dir %{instroot}/%{cmsplatf}/%{pkgcategory}/%{pkgname}/
"""}

COMPILER_DETECTION = { "gcc": "gcc -v 2>&1 | grep version | sed -e \'s|.*\\([0-9][.][0-9][.][0-9]\\).*|\\1|\'",
"icc": "echo no detection callback for icc."}


# Preambles. %dynamic_path_var is defined in rpm-preamble.

INITENV_PREAMBLE = [
("CMD_SH", "if", "[ -f %i/etc/profile.d/dependencies-setup.sh ]; then . %i/etc/profile.d/dependencies-setup.sh; fi"),
("CMD_CSH", "if", "( -f %i/etc/profile.d/dependencies-setup.csh ) source %i/etc/profile.d/dependencies-setup.csh; endif"),
("SETV", "%(uppername)s_ROOT", "%i"),
("SETV", "%(uppername)s_VERSION", "%v"),
("SETV", "%(uppername)s_REVISION", "%pkgrevision"),
("SETV", "%(uppername)s_CATEGORY", "%pkgcategory"),
("+PATH", "PATH", "%i/bin"),
("+PATH", "%%{dynamic_path_var}", "%i/lib")]

DEFAULT_PREAMBLE = """
"""

DEFAULT_DESCRIPTION_PREAMBLE = """
"""

DEFAULT_PREP_PREAMBLE = """
%initenv
[ -d %i ] && chmod -R u+w %i
rm -fr %i
"""

DEFAULT_BUILD_PREAMBLE = """
%initenv
"""

DEFAULT_INSTALL_PREABLE = """
mkdir -p %i
mkdir -p %_rpmdir
mkdir -p %_srcrpmdir
%initenv
"""

DEFAULT_PRE_PREAMBLE = """
if [ X"$(id -u)" = X0 ]; then
    echo "*** CMS SOFTWARE INSTALLATION ABORTED ***"
    echo "CMS software cannot be installed as the super-user."
    echo "(We recommend reading a unix security guide)."
    exit 1
fi
"""

DEFAULT_POST_PREAMBLE = """
if [ "X$CMS_INSTALL_PREFIX" = "X" ] ; then CMS_INSTALL_PREFIX=$RPM_INSTALL_PREFIX; export CMS_INSTALL_PREFIX; fi
%{relocateConfig}etc/profile.d/init.sh
%{relocateConfig}etc/profile.d/init.csh
"""
DEFAULT_PREUN_PREAMBLE = """
"""
DEFAULT_POSTUN_PREAMBLE = """
if [ "X$CMS_INSTALL_PREFIX" = "X" ] ; then CMS_INSTALL_PREFIX=$RPM_INSTALL_PREFIX; export CMS_INSTALL_PREFIX; fi
"""

DEFAULT_FILES_PREAMBLE = """
%%defattr(-, root, root)
"""

COMMANDS_SH = {"SETV":      """%(var)s="%(value)s"\n""",
               "SET":       """export %(var)s="%(value)s";\n""",
               "+PATH":     """[ ! -d %(value)s ] || export %(var)s="%(value)s${%(var)s:+:$%(var)s}";\n""",
               "UNSET":     """unset %(var)s || true\n""",
               "CMD":       """%(var)s %(value)s\n""",
               "CMD_SH":    """%(var)s %(value)s\n""",
               "CMD_CSH":   "",
               "ALIAS":     """alias %(var)s="%(value)s"\n""",
               "ALIAS_CSH": "",
               "ALIAS_SH":  """alias %(var)s="%(value)s"\n"""}

COMMANDS_CSH = {"SETV":     """set %(var)s="%(value)s"\n""",
                "SET":      """setenv %(var)s "%(value)s"\n""",
                "+PATH":    """if ( -d %(value)s ) then\n"""
                            """  if ( ${?%(var)s} ) then\n"""
                            """    setenv %(var)s "%(value)s:$%(var)s"\n"""
                            """  else\n"""
                            """    setenv %(var)s "%(value)s"\n"""
                            """  endif\n"""
                            """endif\n""",
                "UNSET":    """unset %(var)s || true\n""",
                "CMD":      """%(var)s %(value)s\n""",
                "CMD_SH":   "",
                "CMD_CSH":  """%(var)s %(value)s\n""",
                "ALIAS":    """alias %(var)s "%(value)s"\n""",
                "ALIAS_SH": "",
                "ALIAS_CSH":"""alias %(var)s "%(value)s"\n"""}

SPEC_HEADER = """
%%define pkgname        %(name)s
%%define pkgversion     %(version)s
%%define pkgcategory    %(group)s
%%define cmsroot        %(workDir)s
%%define instroot       %(workDir)s/%(tempDirPrefix)s/BUILDROOT/%(checksum)s%(installDir)s
%%define realversion    %(realVersion)s
%%define gccver         %(compilerRealVersion)s
%%define compilerRealVersion %(compilerRealVersion)s
%%define pkgrevision    %(pkgRevision)s
%%define pkgreqs        %(pkgreqs)s
%%define directpkgreqs	%(directpkgreqs)s
%%define specchecksum   %(checksum)s
%%define cmscompiler	%(compilerName)s
%%define cmsbuildApiVersion 1
%%define installroot    %(installDir)s
%%define tempprefix     %(tempDirPrefix)s
Name: %(group)s+%(name)s+%(version)s
Group: %(group)s
Version: %(rpmVersion)s
Release: %(pkgRevision)s
License:  "As required by the orginal provider of the software."
Summary: %(summary)s SpecChecksum:%(checksum)s
%(requiresStatement)s
Packager: CMS <hn-cms-sw-develtools@cern.ch>
Distribution: CMS
Vendor: CMS
Provides: %(group)s+%(name)s+%(version)s
Obsoletes: %(group)s+%(name)s+%(version)s
Prefix: %(installDir)s
"""

DEFAULT_INSTALL_POSTAMBLE="""
# Avoid pkgconfig dependency.  Notice you still need to keep the rm statement
# to support architectures not being build with cmsBuild > V00-19-XX
%if "%{?keep_pkgconfig:set}" != "set"
if [ -d "%i/lib/pkgconfig" ]; then rm -rf %i/lib/pkgconfig; fi
%endif

# Do not package libtool and archive libraries, unless required.
%if "%{?keep_archives:set}" != "set"
# Don't need archive libraries.
rm -f %i/lib/*.{l,}a
%endif

# Strip executable / paths which were specified in the strip_files macro.
%if "%{?strip_files:set}" == "set"
for x in %strip_files
do
  if [ -e $x ]
  then
    find $x -type f -perm -a+x -exec %strip {} \;
  fi 
done
%endif

# remove files / directories which were specified by the drop_files macro.
%if "%{?drop_files:set}" == "set"
for x in %drop_files
do
  if [ -e $x ]; then rm -rf $x; fi
done
%endif

case %{cmsplatf} in
    osx* )
        for x in `find %{i} -type f -perm -u+x | grep -v -e "[.]pyc"`; 
        do 
            if [ "X`file --mime $x | sed -e 's| ||g' | cut -d: -f2 | cut -d\; -f1`" = Xapplication/octet-stream ]
            then
              chmod +w $x
              old_install_name=`otool -D $x | tail -1 | sed -e's|:$||'`
              new_install_name=`basename $old_install_name`
              install_name_tool -change $old_install_name $new_install_name -id $new_install_name $x
              # Make sure also dependencies do not have an hardcoded path.
              for dep in `otool -L $x | sed -e"s|[^\\t\\s ]*%{instroot}|%{instroot}|" | grep -e '^/' | sed -e's|(.*||'`
              do
                install_name_tool -change $dep `basename $dep` $x
              done
              chmod -w $x
            fi
        done
    ;;
    * )
    ;;
esac
"""

DEFAULT_PREP_POSTAMBLE="""
"""

DEFAULT_BUILD_POSTAMBLE="""

# make sure that at least an empty file list does exist
touch %_builddir/files
"""

class CacheProxy (object):
    def __init__ (self, cache, decorator):
        """ This object is responsible for caching object but keeping in mind the fact that
            different architectures will
        """
        self.__cache = cache
        self.__decorator = decorator

    def __getitem__ (self, name):
        return self.__cache.__getitem__ (self.__decorator (name))
        
    def __setitem__ (self, name, item):
        self.__cache.__setitem__ (self.__decorator (name), item)

    def has_key (self, name):
        return self.__cache.has_key (self.__decorator (name))
        
class ArchitectureDecorator (object):
    def __init__ (self, architecture):
        self.__architecture = architecture
    def __call__ (self, name):
        return self.__architecture + name

global checksums_cache
checksums_cache = {}

# Silence the deprecation warning until we move completely to 2.6.X
import warnings
warnings.filterwarnings("ignore",category=DeprecationWarning)
try:
  from md5 import new as md5adder
except ImportError:
  from hashlib import md5 as md5adder

def calculateHumanReadableVersion (pkg):
    # This takes care of converting a checksum to something human-readable.
    #  * If the package has a revision different than 1, we assume that it is
    #    an old style one and we return the realVersion as version.
    #  * If the checksum is available in the DB, we use the tag, rather than 
    #    the checksum.
    #  * If the checksum is not in the DB, but an official tag is requested, 
    #    we associate the checksum to the tag, possibly adding an incremental 
    #    number to it if a given package/version already uses it.

    if not isDefaultRevision(pkg.pkgRevision):
# lange - 080727 -- to get a revision the user will just have to specify the full
# version including the tag. Otherwise we get the tag no matter what..
#
#        if pkg.options.tag:
#            return "%s-%s" % (pkg.realVersion, pkg.options.tag)
        return pkg.realVersion
    log ("%s has checksum %s" % (pkg.name, pkg.checksum), DEBUG)
    tag = tags_cache.getTag (pkg)
    if tag != None:
        log ("%s is aliased to \'%s\', using \'%s\'" % (pkg.checksum, tag, tag), DEBUG)
    elif pkg.options.tag:
        pkgInfo = PkgInfo (pkg)
        tag = tags_cache.requestTag (pkgInfo, pkg.options.tag)
        log ("Attempt to assign %s to \'%s\'" % (pkg.checksum, tag), DEBUG)
    else:
        tag = pkg.checksum
    if not tag:
        return "%s" % pkg.realVersion
    else:
        return "%s-%s" % (pkg.realVersion , tag)

def specFilename (opts, pkgName):
    return join (abspath (opts.cmsdist), "%s.spec" % pkgName)

def redefineMacro (name, value, maxLength=8000, initCount=0):
    exSpec = ""
    value  = re.sub("\s+", " ",value.strip())
    values = splitMacroLine(value, maxLength)
    if len(values)>1:
        value = ""
        for v in values:
            exSpec += "%%define %s%d %s\n" % (name, initCount, v)
            value += "%%{%s%d} " % (name, initCount)
            initCount += 1
        subexSpec, value =  redefineMacro(name, value, maxLength, initCount)
        exSpec += subexSpec
    return (exSpec, value)
    
def splitMacroLine (value, maxLength=8000):
    nvalue = [ value ]
    if len(value)>maxLength:
        nvalue = []
        while len(value) > maxLength:
            xIndex = value.find(" ",maxLength)
            if xIndex == -1: break
            nvalue.append(value[0:xIndex])
            value  = value[xIndex+1:]
        nvalue.append(value)
    return nvalue

class ReadOnlyDict (dict):
    class PermissionError (Exception):
        def __init__ (self):
            Exception.__init__ (self, "Read only dict, cannot set its items")
    def __setitem__ (self, key, value):
        raise ReadOnlyDict.PermissionError ()

class HeaderMatchingRegexps (object):
    def __init__ (self):
        self.REQUIRES_REGEXP = re.compile ("^Requires: (.*)")
        self.REMOTE_SOURCE_REGEXP = re.compile ("^[Ss]ource[0-9]*: (.*:.*/.*)")
        self.REMOTE_PATCH_REGEXP = re.compile ("^[Pp]atch[0-9]*: (.*:.*/.*)")
        self.LOCAL_SOURCE_REGEXP = re.compile ("^[Ss]ource[0-9]*: (.*)")
        self.LOCAL_PATCH_REGEXP = re.compile ("^[Pp]atch[0-9]*: (.*)")
        self.BUILD_REQUIRES_REGEXP = re.compile ("^BuildRequires: (.*)")

class CmsOSDumper (object):
    def __init__ (self, cmsdistPath):
        cmsosFilename = join (abspath (cmsdistPath), "cmsos.file")
        cmsosFile = open (cmsosFilename)
        self.__cmsos = cmsosFile.read ()
        cmsosFile.close ()

    def dump (self, sourcedir):
        destFilename = join (sourcedir, "cmsos")
        log ("Copying cmsos.file to %s" % destFilename, DEBUG)
        destFile = file (destFilename, 'w')
        destFile.write (self.__cmsos)
        destFile.close ()

class PackageFactory (object):
    def __init__ (self, options, cmsosDumperClass=CmsOSDumper):
        self.__package_cache = {}
        self.__requires_cache = {}
        self.__syntax = MetaSpecSyntax ()
        self.__options = options
        self.__cacheKeyDecorator = ArchitectureDecorator (options.architecture)
        self.__preamble = self.__getPreamble ()
        self.__sectionOptions = ReadOnlyDict ({"": "",
                        "%%description": "",
                        "%prep": "",
                        "%build": "",
                        "%install": "",
                        "%pre": "",
                        "%post": "",
                        "%preun": "",
                        "%postun": "",
                        "%files": "-f %_builddir/files"})
        self.__sectionPreambles = ReadOnlyDict ({"": DEFAULT_PREAMBLE,
                        "%%description": DEFAULT_DESCRIPTION_PREAMBLE,
                        "%prep": DEFAULT_PREP_PREAMBLE,
                        "%build": DEFAULT_BUILD_PREAMBLE,
                        "%install": DEFAULT_INSTALL_PREABLE,
                        "%pre": DEFAULT_PRE_PREAMBLE,
                        "%post": DEFAULT_POST_PREAMBLE,
                        "%preun": DEFAULT_PREUN_PREAMBLE,
                        "%postun": DEFAULT_POSTUN_PREAMBLE,
                        "%files": DEFAULT_FILES_PREAMBLE})

        self.__sectionPostambles = {"": "",
                        "%%description": "",
                        "%prep": DEFAULT_PREP_POSTAMBLE,
                        "%build": DEFAULT_BUILD_POSTAMBLE,
                        "%install": DEFAULT_INSTALL_POSTAMBLE,
                        "%pre": "",
                        "%post": "",
                        "%preun": "",
                        "%postun": "",
                        "%files": ""}
        self.__postprocessingRules = [(re.compile ("%\{n\}"), "%{pkgname}"),
                                       (re.compile ("%\{v\}"), "%{pkgversion}"),
                                       (re.compile ("%\{i\}"), "%{pkginstroot}"),
                                       (re.compile ("%n$"), "%{pkgname}"),
                                       (re.compile ("%v$"), "%{pkgversion}"),
                                       (re.compile ("%i$"), "%{pkginstroot}"),
                                       (re.compile ("%n([^_A-Za-z0-9])"), "%{pkgname}\\1"),
                                       (re.compile ("%v([^_A-Za-z0-9])"), "%{pkgversion}\\1"),
                                       (re.compile ("%i([^_A-Za-z0-9])"), "%{pkginstroot}\\1"),
                                       (re.compile ("^Source:"), "Source0:"),
                                       (re.compile ("^Patch:"), "Patch0:")]
        self.__cmsosDumper = cmsosDumperClass (options.cmsdist)
        self.__headerMatchingRegexp = HeaderMatchingRegexps ()

    def __getPreamble (self):
        try:
            filename = join (self.__options.cmsdist, "rpm-preamble.file")
            return open (filename).read ()
        except:
            raise FileNotFound (filename)
            
    def create (self):
        pkg = Package (self.__options)
        pkg._Package__syntax = self.__syntax
        pkg._Package__packageCache = CacheProxy (self.__package_cache, 
                                                 self.__cacheKeyDecorator)
        pkg._Package__requiresCache = CacheProxy (self.__requires_cache, 
                                                  self.__cacheKeyDecorator)
        pkg._Package__preamble = self.__preamble
        pkg._Package__sectionOptions = self.__sectionOptions
        pkg._Package__sectionPreambles = self.__sectionPreambles
        pkg._Package__sectionPostambles = copy.deepcopy (self.__sectionPostambles)
        pkg._Package__cmsosDumper = self.__cmsosDumper
        pkg._Package__headerMatchingRegexp = self.__headerMatchingRegexp
        pkg._Package__factory = self
        return pkg

    def createWithSpec (self, pkgName):
        pkg = self.create ()
        filename = specFilename (self.__options, pkgName)
        log ("Creating package using spec file: %s" % filename, DEBUG)
        try:
            specLines = open (filename).readlines ()
        except IOError, e:
            raise FileNotFound (filename)            
        pkg.initWithSpec (specLines)
        return pkg

    def postProcessSpec (self, spec):
        for regexp, subst in self.__postprocessingRules:
            spec = regexp.sub (subst, spec)
        return spec

    def expandSubpackages (self, packages):
        """ Expand the packages (and their dependencies)'s subpackages.
        """
        out = set()
        for pkg in packages:
            out.add(pkg)
            out.update(pkg.subpackages)
        return sorted(out)

class MetaSpecSyntax (object):
    SPEC_HEADER = property (lambda self : self.__SPEC_HEADER)
    IMPORT      = property (lambda self : self.__IMPORT)
    BUILDIF     = property (lambda self : self.__BUILDIF)
    INITENV     = property (lambda self : self.__INITENV)
    REVISION    = property (lambda self : self.__REVISION)
    SUBPACKAGE  = property (lambda self : self.__SUBPACKAGE)

    def __init__ (self):
        self.__SPEC_HEADER = re.compile ("^### RPM[ ]*([^ ]*)\s*([^ ]*)\s*(.*)")
        self.__IMPORT      = re.compile ("^## IMPORT (.*)")
        self.__BUILDIF     = re.compile ("^## BUILDIF (.*)")
        self.__REVISION    = re.compile ("^## REVISION (.*)")
        self.__SUBPACKAGE  = re.compile (r'^##\s+SUBPACKAGE\s+([\w+-]+)(\s+IF\s+(%[\w+-]+))?\s*$', re.M)

        commands = "|".join (COMMANDS_SH.keys ()).strip ("|").replace ("+", 
                                                                       "[+]")
        initenvStr = "^## INITENV\s+(%s)\s+([^\s]*)\s+(.*)" % commands
        self.__INITENV = re.compile (initenvStr)

class BuilderAction (object):
    def __init__ (self, pkg, parent=None, prevAlternative=None):
        """ This is a base class for an action performed by the builder.
            pkg is the payload for the action.
            parent is an action to be completed/completable before being able 
            to execute this one.
            prevAlternative is an alternative action that has precedence on
            this one.
        """
        self.pkg = pkg
        self.prevAlternative = prevAlternative
        self.parent = parent
        self.__cachedRun = None
        self.__cachedExpectedResults = None
        self.__safeRecursion = 0
    
    def run (self):
        """ Actually run a command.
        """
        assert (False)
        
    def dryRun (self):
        """ Returns True if the action should been executed, False if not.
        """
        # If we already have checked this, do not run the check again.
        if self.__cachedRun:
            return self.__cachedRun
        self.__safeRecursion += 1
        if self.__safeRecursion > 100:
            traceback.print_stack ()
            sys.exit (1)

        actionName = self.actionName
        pkgName = self.pkg.pkgName ()
        # Run the check and cache the results but actually return the correct
        # value only if prerequisites are fulfilled and there is no better 
        # alternative.
        self.__cachedRun = self.doDryRun ()

        # Make sure that all the prerequisites can be executed.
        log ("Checking prerequisites for %(actionName)s %(pkgName)s" % locals (), 
             DEBUG)
        tmpParent = self.parent
        assert (tmpParent != self)
        while tmpParent:
            parentName = tmpParent.actionName
            if tmpParent.dryRun () == False:
                self.cannotRun ("""Cannot run '%(actionName)s' for package %(pkgName)s because 
                    dependending on action '%(parentName)s' which will not be executed.""" % locals ())                
                self.__cachedRun = False
                return False
            tmpParent = tmpParent.parent
        log ("All the prerequisites for %(actionName)s %(pkgName)s are there" % locals (), DEBUG)

        # Make sure that none of the better alternatives can be executed
        log ("Checking altenatives to %(actionName)s %(pkgName)s" % locals (), 
             DEBUG)
        tmpAlternative = self.prevAlternative
        assert (tmpAlternative != self)
        while tmpAlternative:
            if tmpAlternative.dryRun () == True:
                alternativeName = tmpAlternative.actionName
                self.cannotRun ("""Cannot run '%(actionName)s' for package %(pkgName)s because 
                                   alternative action '%(alternativeName)s' has priority.""" % locals ())
                self.__cachedRun = False
                return False
            tmpAlternative = tmpAlternative.prevAlternative
        log ("No better alternative to %(actionName)s for %(pkgName)s." % locals (), DEBUG)
        return self.__cachedRun

    def cannotRun (self, message, level=NORMAL):
        """ Use to specify why you couldn't dryRun something.
        """
        self.cannotRunMessage = message
        self.level = level

    def expectedResults (self):
        if not self.__cachedExpectedResults:
            self.__cachedExpectedResults = self.doExpectedResults ()
        return self.__cachedExpectedResults
        
    def doExpectedResults (self):
        """ Returns True if the command could run as expected with the correct
            outputs being produced, False otherwise.
        """
        assert (False and "Please implement doExpectedResults in derived class.")
        
    def nonExpectedExecution (self):
        """ Return a string to be printed when the execution should have not
            happened, but it did.
        """
        assert (False)
    def missingExecution (self):
        """ Returns a string to be printed when the execution should have
            happened, but it did not.
        """
        assert (False)

    def producesStuffToUpload (self):
        """ Can be overridden to return True if the execution of the action 
            will produce stuff that is then uploaded.
        """
        return False
    actionName = property (lambda self : self._actionName)

class SourcesDownload (BuilderAction):
    def __init__ (self, pkg, parent=None, prevAlternative=None):
        BuilderAction.__init__ (self, pkg, parent, prevAlternative)
        self._actionName = "Download sources"

    def doDryRun (self):
        return True
    
    def doExpectedResults (self):
        remoteSourcesRE = re.compile (".*:.*/.*")
        self.files = []
        for source in self.pkg.sources:
            if remoteSourcesRE.match (source):
                self.files.append (join ("SOURCES", self.pkg.pkgdir, 
                                         source.rsplit("/", 1)[1]))
            else:
                self.files.append (join ("SOURCES", self.pkg.pkgdir,
                                         basename(source).rsplit(".", 1)[0]))
        self.missingFiles = [ abspath (filename)
                              for filename in self.files
                              if not exists (filename) ]
        return self.missingFiles == []
    
    def nonExpectedExecution (self):
        unexpectedFiles = "\n".join ([ filename
                                       for filename in self.files
                                       if exists (filename)])
        return "I was not expecting to find the following files:\n %(unexpectedFiles)s" % locals ()
    
    def missingExecution (self):
        missingFiles = "\n".join (self.missingFiles)
        return "I was expecting the following files: \n %(missingFiles)s" % locals ()
    
class InstallFromLocalArea (BuilderAction):
    def __init__ (self, pkg, parent=None, prevAlternative=None):
        BuilderAction.__init__ (self, pkg, parent, prevAlternative)
        self._actionName = "Install from local area"

    def doDryRun (self):
        """ Return True if there are the rpms ready to be installed.
        """
        # If the area is not bootstrapped, this should not run.
        self.cachedRpmName = join (self.pkg.rpmdir, self.pkg.rpmfilename)
        self.officialRpmName = join (abspath ("RPMS"), self.pkg.rpmfilename)
        if not self.pkg.options.bootstrap:
            return False
        # Otherwise make sure that the packages are there.
        filesExist = exists (self.cachedRpmName) and exists (self.officialRpmName)
        return filesExist 
        
    def doExpectedResults (self):
        """We expect to see the rpm installed in the database, and the files in RPMS"""
        # If we ran this action, we expected rpms to be installed in the db and files 
        # to be there.
        if not self.dryRun ():
            return False 
        pkgName = self.pkg.pkgName ()
        error, output = getstatusoutput ("rpm -q %s" % pkgName)
        # Not all the rpm commands set the error correctly.
        if error or "error:" in output:
            return False
        filesExists = exists (self.cachedRpmName) and exists (self.officialRpmName)
        lines = output.strip ("\n").split ("\n")
        tooManyLines = len (lines) != 1
        if tooManyLines and filesExists:
            return False
        firstLine = lines[0].strip ()
        if not firstLine:
            return False
        if firstLine and firstLine.replace (pkgName, "")[0] != "-" and filesExists:
            return False
        return True
    
    def nonExpectedExecution (self):
        pkgName = self.pkg.pkgName ()
        officialRpmName = self.officialRpmName 
        return "%(pkgName)s was installed from %(officialRpmName)s although no installation can happen from the local area." % locals ()
    
    def missingExecution (self):
        pkgName = self.pkg.pkgName ()
        officialRpmName = self.officialRpmName 
        return "I was expecting %(pkgName)s to be installed using %(officialRpmName)s but it was not." % locals ()
        

class InstallFromServer (BuilderAction):
    def __init__ (self, pkg, parent, prevAlternative):
        BuilderAction.__init__ (self, pkg, parent, prevAlternative)
        self.called = False
        self._actionName = 'Install from server'
          
    def doDryRun (self):
        """ Return True if the package is on server
        """
        if not self.pkg.options.bootstrap:
            return False
        self.called = True
        pkgName     = self.pkg.pkgName ()
        pkgInfo     = PkgInfo (self.pkg)
        pkgRealName = pkgInfo.id()
        pkgsWithSameHash = [p for (p,h) in tags_cache.cache.iteritems() if (h == pkgInfo.checksum) and p.startswith(pkgRealName) and (pkgName != p) ]

        # If a file exists on server we return True
        pkgNameApt = self.pkg.pkgName ().replace ("+", "\\+")         
        aptCommand = """apt-cache showpkg %(pkgNameApt)s""" % locals ()
        error, output = getstatusoutput (aptCommand)
        try:
          self.output = output.split("Versions:",1)[1].split('Reverse Depends:')[0]
        except:
          die("Error while parsing %s" % output)
        def evalExistsOnServer ():
            for line in self.output.split ("\n"): 
                if not line.startswith("1-%s." % self.pkg.pkgRevision):
                    continue
                if "var/lib/apt/lists" in line:
                    return True
            return False
        if error:
            return False
        return pkgsWithSameHash or evalExistsOnServer ()
    
    def doExpectedResults (self):
        # In order for a download to be successful we expect:
        # 1) to find it on server
        # 2) to find it on client
        # 3) not to find the rpm in RPMS
        if not self.called:
            return False
        
        def onServerAndLocal ():
            if "var/lib/apt/lists" in self.output and "var/lib/rpm/Packages" in self.output:
                return True
            return False 
        rpmInstalled = onServerAndLocal ()
        rpmFilename = join (abspath (self.pkg.rpmdir), self.pkg.rpmfilename)
        return rpmInstalled and not exists (rpmFilename) 
    
    def nonExpectedExecution (self):
        pkgName = self.pkg.pkgName ()
        return """I was not expecting to download %(pkgName)s from server 
                  but it looks like I can.
                  Probably you need to run:
                  
                  cmsBuild deprecate-local %(pkgName)s """ % locals ()
   
    def missingExecution (self):
       pkgName = self.pkg.pkgName ()
       return """ I was expecting to be able to download %(pkgName)s from server, 
                  but I can't. Are you sure you don't need to rebuild it?""" % locals ()


class BuildPackage (BuilderAction):
    def __init__ (self, pkg, parent=None, prevAlternative=None):
        BuilderAction.__init__ (self, pkg, parent, prevAlternative)
        self._actionName = 'Build Package'
        
    def doDryRun (self):
        """ The only requirement for a package to be built apart from sources 
            being there and no other alternatives (installing from sever) is that
            the specfile for it exists.
            For a subpackage, check the parent's specfile.
        """
        pkg = self.pkg
        if pkg.name == "system-compiler":
            return False
        if pkg.parent:
            specfilename = join (pkg.parent.options.cmsdist, "%s.spec" % pkg.parent.name)
        else:
            specfilename = join (pkg.options.cmsdist, "%s.spec" % pkg.name)
        return exists (specfilename)
    
    def doExpectedResults (self):
        if self.pkg.name == "system-compiler":
            return False
        cachedRpmName = join (self.pkg.rpmdir, self.pkg.rpmfilename)
        return exists (cachedRpmName)
    
    def nonExpectedExecution (self):
        pkgName = self.pkg.pkgName ()
        return "I would have not expected %(pkgName)s to be built, but it was." % locals ()
    
    def missingExecution (self):
        pkgName = self.pkg.pkgName ()
        return "I would have expected a local build of %(pkgName)s but that did not happen." % locals ()
    
    def producesStuffToUpload (self):
        """ A package that gets built will always produce stuff to upload.
        """
        return True

class BuildSystemCompiler (BuilderAction):
    def __init__ (self, pkg, parent=None, prevAlternative=None):
        BuilderAction.__init__ (self, pkg, parent, prevAlternative)
        self._actionName = 'Build system compiler'
        
    def doDryRun (self):
        """ The only case we build the system compiler is the case in which the 
            name matches the dummy package.
        """
        if self.pkg.name != "system-compiler":
            return False
        return self.pkg.options.systemCompiler
    
    def doExpectedResults (self):
        return self.pkg.name == "system-compiler" 
    
    def nonExpectedExecution (self):
        return "I was not expecting to use the system compiler."
    
    def missingExecution (self):
        return "I was expecting to use the system compiler."

class LinkPackageFromCache (BuilderAction):
    def __init__ (self, pkg, parent=None, prevAlternative=None):
        BuilderAction.__init__ (self, pkg, parent=None, prevAlternative=None)
        self._actionName = 'Create links for rpm'
        self.called = False
        
    def doDryRun (self):
        """ We always expect to be able to run the link, if the building
            was possible. Since the building is actually performed by a 
            parent action we always return True and make sure we have a
            BuildAction as parent.
        """
        self.cachedRpmName = join (self.pkg.rpmdir, self.pkg.rpmfilename)
        self.officialRpmName = join (abspath ("RPMS"), self.pkg.rpmfilename)
        self.called = True
        if not exists (self.cachedRpmName):
            return False
        return True
    
    def doExpectedResults (self):
        """ We expect the link to exists, to be readable, and to point to a
            an RPM which has the same checksum as the package associated to
            this action.
        """
        if not self.called:
            return False
        if not exists (self.officialRpmName):
            return False
        checksum = getPkgChecksumFile (self.officialRpmName, self.pkg.options)
        return self.pkg.checksum == checksum

    def nonExpectedExecution (self):
        officialRpmName = self.officialRpmName
        cachedRpmName = self.cachedRpmName
        return """A link was found in %(officialRpmName)s but it points to the
                  wrong rpm %(cachedRpmName)s""" % locals ()
               
    def missingExecution (self):
        officialRpmName = self.officialRpmName
        cachedRpmName = self.cachedRpmName
        return """A link was expected from %(officialRpmName)s to %(cachedRpmName)s, but it not found. """ % locals ()

def detectCompilerVersion (compilerName):
    (error, version) = getstatusoutput(COMPILER_DETECTION[compilerName])
    if error:
        raise UnknownCompiler ()
    return version.strip ("\n")

class ChecksumCalculator (object):
    def __init__ (self):
        self.__adder = md5adder ()
        self.__checksums = {}
    
    def addString (self, string):
        self.__adder.update (string)
    
    def addStrings (self, stringList):
        self.__adder.update ("".join (stringList))

    def addFile (self, filename):
        assert (filename[0] == "/")
        if checksums_cache.has_key (filename):
            self.__checksums[filename] = checksums_cache[filename]
            return
        if not exists (filename):
            raise FileNotFound (filename)
        f = open (filename)
        m = md5adder (f.read ())
        f.close ()
        checksum = m.hexdigest ()
        checksums_cache[filename] = checksum
        self.__checksums[filename] = checksum
        
    def addPkg (self, pkg):
        if pkg.name == "system-compiler":
            filename = pkg.name
        else:
            filename = specFilename (pkg.options, pkg.name)
        self.__checksums[filename] = pkg.checksum
        
    def getChecksum (self):
        items = self.__checksums.items ()
        items.sort ()
        for key, value in items:
            self.__adder.update (value)
        return self.__adder.hexdigest ()

class SubPackage (object):
    def __init__ (self, parent, name):
        self.subname        = name
        self.parent         = parent
        self.subpackages    = []
        self.sources        = []
        #self.updateFromParent()

    def updateFromParent (self):
        self.name           = '%s-%s' % (self.parent.name, self.subname)
        self.group          = self.parent.group
        self.version        = self.parent.version
        self.realVersion    = self.parent.realVersion
        self.pkgRevision    = self.parent.pkgRevision
        self.pkgrel         = self.parent.pkgrel
        self.pkgdir         = self.parent.pkgdir
        self.options        = self.parent.options
        self.cmsplatf       = self.parent.cmsplatf
        self.workDir        = self.parent.workDir
        self.installDir     = self.parent.installDir
        self.tempDirPrefix  = self.parent.tempDirPrefix
        self.builddir       = self.parent.builddir
        self.sourcedir      = self.parent.sourcedir
        self.specdir        = self.parent.specdir
        self.checksum       = self.parent.checksum
        self.requires       = [ self.parent.pkgName() ]
        self.dependencies   = [ self.parent ] + self.parent.dependencies
        self.origDependencies = [ self.parent ] + self.parent.origDependencies
        self.buildDependencies = [ self.parent ] + self.parent.buildDependencies
        self.origBuildDependencies = [ self.parent ] + self.parent.origBuildDependencies
        self.fullDependencies = [ self.parent ] + self.parent.fullDependencies
        self.dependentCounter = 0
        self.patches        = self.parent.patches
        self.rpmdir         = self.parent.rpmdir
        self.rpmfilename    = '%s/%s+%s+%s-%s-%s.%s.rpm' % (self.cmsplatf, self.group, self.name, self.version, '1', self.pkgRevision, self.cmsplatf)
        self.srpmfilename   = self.parent.srpmfilename
        self.pkginstroot    = self.parent.pkginstroot

    def dumpSpecFragment(self):
        importName = 'subpackage-%s.file' % self.subname
        importFilename = join (self.options.cmsdist, importName)
        spec = open (importFilename, 'r').read()
        spec = re.compile(r'^Summary:.*$', re.M).sub(r'\g<0> SpecChecksum:%s' % self.checksum, spec)
        return spec

    def pkgName (self):
        return "%(group)s+%(name)s+%(version)s" % self.__dict__

    def rpmLocation(self):
        # make sure all the information is available, otherwise update from the parent package
        if not self.rpmdir or not self.rpmfilename:
            self.updateFromParent()
        return join(self.rpmdir, self.rpmfilename)

    def dumpCmsos (self):
        self.parent.dumpCmsos()

    def __repr__ (self):
        return "<SubPackage name=%(name)s>" % self.__dict__
    
    def __eq__ (self, other):
        """ Two (sub)packages are the same if they have the same name"""
        return type (other) == SubPackage and self.name == other.name
    
    def __ne__ (self, other):
        return not self.__eq__ (other)

    # FIXME write a single implementation for both Package and SubPackage    
    def __cmp__ (self, other):
        """ Subpackage A is greater than package B if A is a subpackage of B.
            Two subpackages of the same package are sorted by name, 
            while two subpackages of different packages have the same 
            ordering as their parent packages.
            A subpackage and an unrelated package have the same order as the
            subpackage's parent and the other package.
        """
        if self.parent == other:
            return 1
        elif self.parent == other.parent:
            return cmp(self.name, other.name)
        elif other.parent is None:
            return cmp(self.parent, other)
        else:
            return cmp(self.parent, other.parent)


class Package (object):
    tmpspec = property (lambda self : self.__getTmpSpecName ())
    preamble = property (lambda self : self.__preamble)
    sectionOptions = property (lambda self : self.__sectionOptions)
    sectionPreambles = property (lambda self : self.__sectionPreambles)
    sectionPostambles = property (lambda self : self.__sectionPostambles)
    sourcedir = property (lambda self : self.__getSourcedir ())
    specdir = property (lambda self : self.__getSpecdir ())
    pkgdir = property (lambda self : self.__getPkgdir ())

    def __init__ (self, options):
        self.workDir = options.workDir
        self.installDir = options.installDir
        self.tempDirPrefix = options.tempDirPrefix
        # FIXME: actually detect gcc.
        self.compilerRealVersion = ""
        self.name = ""
        self.realVersion = ""
        self.group = ""
        self.version = ""
        self.summary = "CMS Experiment package"
        self.license = "As defined by package owner"
        self.subname = ""
        self.parent = None
        self.subpackages = []
        self.env = ""
        self.urls = []
        self.requires = []
        self.imports = []
        self.buildrequires = []
        self.sources = []
        self.patches = []
        self.pkgreqs = "%{nil}"
        self.directpkgreqs = "%{nil}"
        self.buildCondition = None
        self.builddir = None
        self.pkginstroot = None
        self.pkgrel = None
        self.__specdir = None
        self.cmsplatf = options.architecture 
        self.rpmdir = None
        self.rpmfilename = None
        self.srpmfilename = None
        self.pkgRevision = '1'
        self.specPreHeader = ""
        #For online: pkgRevision is always architecture
        if isOnline(self.cmsplatf): self.pkgRevision += '.'+self.cmsplatf
        self.rpmVersion = "1"

        self.requiresStatement = "Requires: gcc"
        self.sections = {}
        self.dependentCounter = 0 
        self.__sectionOptions = None
        self.__sectionPreambles = None
        self.__sectionPostambles =  None

        self.dependencies = []
        self.origDependencies = []
        self.buildDependencies = []
        self.origBuildDependencies = []
        self.fullDependencies = []
        self.origSpec = []
        self.spec = []
        self.checksum = "%{nil}"
        self.options = options
        self.status = None
        self.__packageCache = None 
        self.__requiresCache = None
        self.__syntax = None
        self.__preamble = ""
        self.buildRequireToolfile = False
   
    def rpmLocation(self):
        return join(self.rpmdir, self.rpmfilename)
    
    def createDefaultSections (self):
        if self.options.skipPreInstall == True:
            log  ("Skip preinstall check: " + str(self.options.skipPreInstall), DEBUG)
            replacement="""
# Built with an option to skip CMS default pre-install checks
"""
            DEFAULT_SECTIONS['%pre']=replacement
        for section in DEFAULT_SECTIONS.keys ():
            self.sections[section] = {}
            self.sections[section][''] = DEFAULT_SECTIONS[section]
            #FIXME: This is to avoid trigger a rebuild. This overrides the %files section
            #For any release series which is going to trigger a rebuild please fix DEFAULT_SECTIONS[%file]
            #to have only %{installroot}/ and remove next 2 lines.
            if section == "%files":
                self.sections[section][''] = '%{installroot}/'
            self.sections[section][''] += self.sectionPostambles[section]
    
    def initWithSpec (self, specLines):
        """ This parses the spec files and creates a structure with its contents.
        """
        self.origSpec = specLines
        self.group, self.name, self.realVersion = parseRPMLine (self.origSpec, self.options)
        self.version = self.realVersion
        self.parseRevision ()
        compilerName = self.options.compilerName
        self.compilerName = compilerName
        self.expandSpec ()
        if self.name == compilerName:
            self.requiresStatement = ""
            self.compiler = self
        else:
            if self.__packageCache.has_key (compilerName):
                self.compiler = self.__packageCache[compilerName]
            else:
                if self.options.systemCompiler == True:
                    self.compiler = self.__factory.create ()
                    self.compiler.name = "system-compiler"
                    version = detectCompilerVersion (compilerName)
                    self.compilerRealVersion = version
                    self.compiler.realVersion = version
                else:
                    self.compiler = self.__factory.createWithSpec (compilerName)
                    self.compilerRealVersion = self.compiler.realVersion
                self.__packageCache[compilerName] = self.compiler
            if self.options.systemCompiler == False and parseNoCompilerLine(self.spec) == False:
                self.dependencies.append (self.compiler)
                self.origDependencies.append (self.compiler)
        self.buildRequireToolfile = parseBuildRequireToolfileLine(self.spec)
        self.fullDependencies += self.dependencies
        self.compilerRealVersion = self.compiler.realVersion
        self.generateInitEnv ()
        self.createDefaultSections ()
        self.parseSections ()
        self.dumpSpec ()
        self.dumpCmsos ()
        self.origRequires, self.sources, self.patches, self.origBuildRequires = self.getRequiresAndSources ()
        self.parseRequires ()
        self.dumpSpec ()
        # FIXME: saveSpec should really be done once.
        createDirs (architecture=self.cmsplatf)
        self.calculateChecksum ()
        self.rewriteRequires ()
        self.dumpSpec ()
        prepareSaveSpec (self)
        saveSpec (self)
        result = self.rpmEvalStrings ("%pkgrel", 
            "%pkginstroot", 
            "%_builddir", 
            "%_specdir",
            "%_rpmdir", 
            "%_rpmfilename",
            "%_srpmfilename")
        assert (result)
        try:
            [self.pkgrel, self.pkginstroot, self.builddir, 
             self.__specdir, self.rpmdir, 
             self.rpmfilename, self.srpmfilename ] = result
        except ValueError, e:
            log ("FATAL: Error unpacking results: \n%s" % result)
            raise e
        # update subpackages
        for subpackage in self.subpackages:
            subpackage.updateFromParent()
        
    def parseRevision (self):
        for line in self.origSpec:
            match = self.__syntax.REVISION.match (line)
            if match:
                self.pkgRevision = match.group(1)
                if isOnline(self.cmsplatf): self.pkgRevision += '.'+self.cmsplatf
                return True
        return False

    def expandSpec (self):
        """This function is responsible for finding and parsing all the IMPORT and SUBPACKAGE directives for a given spec.
        """
        imports = []
        subpackages = {}
        self.subpackages = []
        for line in self.origSpec:
            imatch = self.__syntax.IMPORT.match (line)
            smatch = self.__syntax.SUBPACKAGE.match (line)
            if imatch:
                imports.append (imatch.group (1).strip (" \n") + ".file")
            elif smatch:
                subpackages[smatch.group(1)] = smatch.group(3)
            else:
                self.spec.append (line)
        try:
            for importName in imports:
                importFilename = join (self.options.cmsdist, importName)
                self.spec.extend (open (importFilename).readlines ())
                self.imports.append (importFilename) 
        except IOError, e:
            raise FileNotFound (importFilename)

        # Check for sub-packages after all processing all the imports
        for subpackageName in subpackages:
            condition = subpackages[subpackageName]
            if not condition or self.isSymbolDefined(condition):
                self.subpackages.append( SubPackage(self, subpackageName) )

    def isSymbolDefined(self, symbol):
        """This function checks if a symbol is defined in the current spec file. That is, if rpmbuild evaluates %symbol to a non-empty string. 
        Note: I *think* that only the symbols defined in the main section are visible here.
        """
        tmp = copy.copy(self)
        tmp.createDefaultSections()
        tmp.parseSections()
        value = tmp.rpmEvalStrings(symbol)
        isDefined = (value != symbol)
        return isDefined

    def parseSections (self):
        """ This helper method is responsible for parsing the spec and create subdivide sections found inside it.
        """
        currentSection = ''
        currentSubSection = ''
        SECTION_MATCH="(%s)[$\s]+(.*)" % "|".join (self.sections.keys ()).strip ('|')
        sectionRe = re.compile (SECTION_MATCH)
        for line in self.spec:
            match = sectionRe.match (line)
            if match:
                # Close the previous section and open a new one.
                self.sections[currentSection][currentSubSection] += self.sectionPostambles[currentSection]
                currentSection, currentSubSection = match.groups ()
                self.sections[currentSection][currentSubSection] = self.sectionPreambles[currentSection]
            else:
                self.sections[currentSection][currentSubSection] += line
        self.sections[currentSection][currentSubSection] += self.sectionPostambles[currentSection]
        
        # If no BUILDIF conditions we are done. 
        # Otherwise insert the build condition everywhere, so that sections 
        # are not actually executed.
        if not self.buildCondition:
            return
        
        for section in ["%install", "%build"]:
            for subsection in self.sections[section].keys ():
                old = self.sections[section][subsection]
                self.sections[section][subsection] = old.replace ("%initenv", 
                                                                  "%s || exit 0\n%%initenv\n" % self.buildCondition)
        for section in ["%pre", "%post", "%preun", "%postun"]:
            for subsection in self.sections[section].keys ():
                old = self.sections[section][subsection]
                self.sections[section][subsection] = "%s || exit 0\n" % self.buildCondition + old

    def calculateChecksum (self):
        """ 
        """
        checksumCalculator = ChecksumCalculator ()
        
        if self.checksum != "%{nil}":
            return
        # In the case --use-system-compiler option is specified, the compiler itself
        # is called "system-compiler" and the checksum is given by its version rather
        # than the spec.
        # FIXME: add more sensible stuff to the checksum, besides the version (maybe the 
        # gcc specfile???) 
        if self.name == "system-compiler":
            checksumCalculator.addString (self.compilerRealVersion)
            self.checksum = checksumCalculator.getChecksum ()
            self.version = calculateHumanReadableVersion (self)
            return 

        checksumCalculator.addStrings (self.origSpec)
        checksumCalculator.addStrings (DEFAULT_SECTIONS.values ())
        checksumCalculator.addStrings (self.sectionOptions.values ())    
        checksumCalculator.addStrings (self.sectionPreambles.values ())    
        checksumCalculator.addStrings (self.sectionPostambles.values ())
        
            
        for filename in self.imports:
            checksumCalculator.addFile (abspath (filename))

        for subpackage in self.subpackages:
            checksumCalculator.addString( subpackage.dumpSpecFragment() )

        for pkg in self.dependencies:
            checksumCalculator.addPkg (pkg)
        for pkg in self.buildDependencies:
            checksumCalculator.addPkg (pkg)
        remotefileRE = re.compile (".*:.*/.*")
        for patch in self.patches:
            if remotefileRE.match (patch):
               continue
            filename = join (abspath (self.options.cmsdist), patch)
            checksumCalculator.addFile (filename)
        for source in self.sources:
            if remotefileRE.match (source):
               continue 
            filename = join (abspath (self.options.cmsdist), source)
            if exists (filename):
                checksumCalculator.addFile (filename)
        # FIXME: should we add downloaded sources checksums to the global checksum? This is actually
        #        tricky because at the moment the sources are downloaded *after* the checksums
        #        are calculated and they are put in a directory which has the global checksum in
        #        the name. We could fetch sources first and drop the global checksum from the 
        #        sources download dir, but this means that we will be using always the same 
        #        directory for the sources, even if they might be different (for example when downloading them from cvs). 
        self.checksum = checksumCalculator.getChecksum ()
        self.version = calculateHumanReadableVersion (self)

    def generateRequires (self, dependencies, origDependencies):
        requiresPkgs = [ "%s+%s+%s" % (pkg.group, pkg.name, pkg.version) 
                         for pkg in dependencies
                         if pkg.name != "system-compiler" ]
        pkgreqsPkgs = [ "%s/%s/%s " % (pkg.group, pkg.name, pkg.version) 
                        for pkg in dependencies
                        if pkg.name != "system-compiler" ]
        directpkgreqsPkgs = [ "%s/%s/%s " % (pkg.group, pkg.name, pkg.version) 
                        for pkg in origDependencies
                        if pkg.name != "system-compiler" ]
        requiredtoolsPkgs = [ pkg.name
                              for pkg in dependencies
                              if pkg.name != "system-compiler" ]
        requires = " ".join (requiresPkgs)
        pkgreqs = " ".join (pkgreqsPkgs)
        directpkgreqs = " ".join (directpkgreqsPkgs)
        requiredtools = " ".join (requiredtoolsPkgs)
        if not pkgreqs:       pkgreqs       = "%{nil}"
        if not directpkgreqs: directpkgreqs = "%{nil}"
        if not requiredtools: requiredtools = "%{nil}"
        return (requires, pkgreqs, directpkgreqs, requiredtools)

    def rewriteRequires (self):
        """ This rewrites both the Requires line to specify dependencies and
            sets the pkgreq to point to their path.
        """
        requires, pkgreqs, directpkgreqs, requiredtools = self.generateRequires(self.dependencies, self.origDependencies)
        bldrequires, bldpkgreqs, blddirectpkgreqs, bldrequiredtools = self.generateRequires(self.origBuildDependencies, self.origBuildDependencies)
        allDepsPkgs = " ".join([ "%s/%s/%s " % (pkg.group, pkg.name, pkg.version) 
                        for pkg in self.fullDependencies
                        if pkg.name != "system-compiler" ])
        if not allDepsPkgs: allDepsPkgs = "%{nil}"
        #After rewrite of Requires we split pkgeqs and directpkgreqs to avoid RPM macro length limit
        specPre1, pkgreqs          = redefineMacro("pkgreqs",           pkgreqs)
        specPre2, requiredtools    = redefineMacro("requiredtools",     requiredtools)
        specPre3, directpkgreqs    = redefineMacro("directpkgreqs",     directpkgreqs)
        specPre4, bldpkgreqs       = redefineMacro("buildpkgreqs",      bldpkgreqs)
        specPre5, bldrequiredtools = redefineMacro("buildrequiredtools",bldrequiredtools)
        specPre6, blddirectpkgreqs = redefineMacro("builddirectpkgreqs",blddirectpkgreqs)
        specPre7, allDepsPkgs      = redefineMacro("allpkgreqs",        allDepsPkgs)

        requires      = "\nRequires: ".join(splitMacroLine(requires))
        bldrequires   = "\nBuildRequires: ".join(splitMacroLine(bldrequires))
        reqStatement  = "%%define requiredtools %(requiredtools)s\n%%define buildrequiredtools %(bldrequiredtools)s\n" 
        reqStatement += "%%define buildpkgreqs %(bldpkgreqs)s\n%%define builddirectpkgreqs %(blddirectpkgreqs)s\n"
        reqStatement += "%%define allpkgreqs   %(allDepsPkgs)s\n"
        if requires:
            reqStatement += "Requires: %(requires)s\n"
        if bldrequires:
            reqStatement += "BuildRequires: %(bldrequires)s\n"
        self.pkgreqs           = pkgreqs
        self.directpkgreqs     = directpkgreqs
        self.requiresStatement = reqStatement % locals()
        self.specPreHeader     = specPre1 + specPre2 + specPre3 + specPre4 + specPre5 + specPre6 + specPre7
        
    def updateChildDependencies (self, pkg, dependencies, pkgdeps):
        if not pkg in dependencies: dependencies.insert (0, pkg)
        for subdep in pkgdeps:
            if not subdep in dependencies: dependencies.insert (0, subdep)
        return

    def updateDependencies (self, require, origDependencies, dependencies, buildRequires=False):
        if self.__packageCache.has_key (require):
            pkg = self.__packageCache[require]
        else:
            pkg = self.__factory.createWithSpec (require)
            self.__packageCache[require] = pkg
        origDependencies.insert (0, pkg)
        self.updateChildDependencies(pkg, dependencies, pkg.dependencies)
        if buildRequires:
            self.updateChildDependencies(pkg, dependencies, pkg.buildDependencies)

    def parseRequires (self):
        """This function is responsible for finding all the required dependencies for a given spec.
           We recursively create dependend Packages or we pick them from the package_cache if needed.
           The dependencies of the dependent Packages are added to the dependency list of the parent,
           so that each Package has the full list of dependent packages. If a dependency is already there
           in the list, it gets "bubbled", so that more fundamental dependencies will come first in the
           list.
        """
        for require in self.origRequires:
            self.updateDependencies(require, self.origDependencies, self.dependencies)
        if self.buildRequireToolfile:
            deps = self.dependencies[:]
            for pkg in deps:
                dep = pkg.name
                if not dep.endswith("-toolfile"):  continue
                if not self.__packageCache.has_key(dep.replace("-toolfile","")): continue
                if not dep in self.origBuildRequires: self.origBuildRequires.append(dep)
                if pkg in self.origDependencies: self.origDependencies.remove(pkg)
                self.dependencies.remove(pkg)
        for require in self.origBuildRequires:
            self.updateDependencies(require, self.origBuildDependencies, self.buildDependencies, True)
        for dep in self.dependencies:
            if dep in self.origBuildDependencies: self.origBuildDependencies.remove(dep)
            if dep in self.buildDependencies:     self.buildDependencies.remove(dep)
        self.fullDependencies = self.dependencies + self.buildDependencies
        for dep in self.fullDependencies:
            dep.dependentCounter += 1
        self.dependencies.sort()
        self.origDependencies.sort()
        self.buildDependencies.sort()
        self.origBuildDependencies.sort()
        self.fullDependencies.sort()

    def dumpSpec (self):
        self.spec = "\n".join([self.specPreHeader, 
                               SPEC_HEADER % self.__dict__, 
                               self.preamble])
        for section in self.sections.keys ():
            for subsection in self.sections[section].keys ():
                sectionContents = self.sections[section][subsection].strip ("\n ")
                if sectionContents:
                    self.spec += "\n\n%s %s %s\n" % (section, subsection, self.sectionOptions[section])
                    self.spec += sectionContents
        self.spec = self.__factory.postProcessSpec (self.spec)

        for subpackage in self.subpackages:
            subpackage.updateFromParent()
            self.spec += '\n\n' + subpackage.dumpSpecFragment()

    def __getTmpSpecName (self):
        # FIXME: make this static? Should not change over the whole period.
        return join (abspath (self.tempDirPrefix), "tmpspec-%s" % self.name)
        
    def rpmEvalStrings (self, *strings):
        self.spec = "\n".join ([self.specPreHeader,
                                SPEC_HEADER % self.__dict__,
                                self.preamble,
                                self.sections[''][''],
                                "%description"] + list (strings)) 
        f = file (self.tmpspec, "w")
        f.write (self.__factory.postProcessSpec (self.spec).replace('%%', '%'))
        f.close ()
        commandPrefix = getCommandPrefix (self.options)
        evalCommand = "%s rpm -q --specfile %s --info %s" % (commandPrefix,
                                                      self.tmpspec, self.options.rpmQueryDefines)
        log (evalCommand, DEBUG)
        error, output = getstatusoutput (evalCommand)
        if error:
            log ("FATAL: malformed spec found while quering it. Command: ")
            log (evalCommand)
            log ("Resulted in:\n\n%s" % output)
            raise MalformedSpec (self.tmpspec)
        log (output, DEBUG)
        #This allows us to build packages which have 'Description' as a part of their name.
        description = re.split ("Description\s*:",output,1)[1]
        results = [line for line in description.split ("\n")][1:]
        return (len (results) == 1 and results[0]) or results

    def __getPkgdir (self):
        return join (self.group, self.name, self.version)
    
    def __getSourcedir (self):
        return join (abspath ("SOURCES"), self.pkgdir)

    def __getSpecdir (self):
        return join (abspath ("SPECS"), self.pkgdir)
    
    def specFilename (self):
        return join (self.specdir, "spec")

    def finalSpecFilename (self):
        return join (abspath ("SPECS"), self.pkgdir, "spec")

    def pkgName (self):
        return "%(group)s+%(name)s+%(version)s" % self.__dict__

    def rpmCachePath (self):
        return join (abspath ("RPMS/cache"), self.checksum, self.rpmfilename)

    def generateInitEnv (self):
        """ Parses the spec file and generates the code for init.sh/init.csh
        """
        self.initSh = """cat <<\EOF_INIT_SH > %i/etc/profile.d/init.sh\n"""        
        self.initCsh = """cat <<\EOF_INIT_CSH > %i/etc/profile.d/init.csh\n"""

        upperNameDict = {"uppername": self.name.upper ().replace ("-", "_")}

        if self.name == "gcc":
            self.initSh += COMMANDS_SH["+PATH"] % {"var": "PATH", "value": "%{i}/bin-real"}
            self.initCsh += COMMANDS_CSH["+PATH"] % {"var": "PATH", "value": "%{i}/bin-real"}
        for command, var, value in INITENV_PREAMBLE:
            self.initSh += COMMANDS_SH[command] % {"var": var % upperNameDict, "value": value}
            self.initCsh += COMMANDS_CSH[command] % {"var": var % upperNameDict, "value": value}            
            
        for line in self.spec:
            match = self.__syntax.INITENV.match (line)
            if match:
                command, var, value = [x.strip (" \t") for x in match.groups ()]
                self.initSh += COMMANDS_SH[command] % {"var": var, "value": value}
                self.initCsh += COMMANDS_CSH[command] % {"var": var, "value": value}
            buildifMatch = self.__syntax.BUILDIF.match (line)
            if not self.buildCondition and buildifMatch:
                self.buildCondition = buildifMatch.group (1)
        self.initSh += "\nEOF_INIT_SH\n"
        self.initCsh += "\nEOF_INIT_CSH\n"
        self.__createProfileDScript = "mkdir -p %i/etc/profile.d\n"
        self.sectionPostambles["%install"] += "\n".join ([self.__createProfileDScript,
                                                          self.initSh,
                                                          self.initCsh])

    def getFinalSpec (self):
        self.dumpSpec ()
        return self.spec
    
    def dumpCmsos (self):
        if not exists (self.sourcedir):
            makedirs (self.sourcedir)
        self.__cmsosDumper.dump (self.sourcedir)
    
    def __repr__ (self):
        return "<Package name=%(name)s>" % self.__dict__
    
    def __eq__ (self, other):
        """ Two packages are the same if they have the same name"""
        return type (other) == Package and self.name == other.name
    
    def __ne__ (self, other):
        return not self.__eq__ (other)
    
    def __cmp__ (self, other):
        """ Package A is greater than package B if
            Package A depends on package B.
            Moreover, if A does not depend B and viceversa, the one with less dependent
            packages greater first. If they have the same number of dependent packages, the
            one with less dependencies is said to be greater. 
        """
        moredeps = cmp (len (self.fullDependencies), len (other.fullDependencies))
        lessdependent = cmp (other.dependentCounter, self.dependentCounter)
        if self in other.fullDependencies:
            return -1
        elif other in self.fullDependencies:
            return 1
        elif lessdependent:
            return lessdependent
        elif moredeps:
            return moredeps
        else:
            return cmp (self.name, other.name)
    
    def getRequiresAndSources (self, subpackage=''):
        if self.__requiresCache.has_key (self.name):
            return self.__requiresCache[self.name]
        spec = file (self.tmpspec, "w")
        text = "\n".join([self.specPreHeader,
                          SPEC_HEADER % self.__dict__,
                          self.__factory.postProcessSpec (self.preamble),
                          "%%description",
                          self.__factory.postProcessSpec (self.sections[''][subpackage])])
        spec.write (text)
        spec.close ()
        deps = []; sources = [] ; patches = []
        localSources = []
        localPatches = []
        buildDeps    = []
        queryCommand = "%s rpm -q --info --specfile %s %s 2>/dev/null" % (getCommandPrefix (self.options),
                                                                  self.tmpspec, self.options.rpmQueryDefines)
        regexps = self.__headerMatchingRegexp
        matchers = [(regexps.REQUIRES_REGEXP, deps),
                    (regexps.REMOTE_SOURCE_REGEXP, sources),
                    (regexps.REMOTE_PATCH_REGEXP, patches),
                    (regexps.LOCAL_SOURCE_REGEXP, localSources),
                    (regexps.LOCAL_PATCH_REGEXP, localPatches),
                    (regexps.BUILD_REQUIRES_REGEXP, buildDeps)]
        for line in popen (queryCommand).readlines ():
            line = re.sub ("[\s]+", " ", line).strip ("\n\t ")
            for rule, target in matchers:
                match = rule.match (line)
                if not match:
                    continue
                target.extend ([element 
                                for element in match.group (1).split ()])
                break
        cmsdistPath = abspath (self.options.cmsdist)
        # If no site is set, simply returns the original dependency name.
        # If site is set, return "site-dep" as a dependency if site-dep.spec
        # exists.
        def siteSpecific(d):
          if not self.options.site:
            return d
          siteDep = self.options.site + "-" + d
          siteSpec = join(cmsdistPath, siteDep + ".spec")
          if exists(join(cmsdistPath, siteDep + ".spec")):
            log("Site specific spec %s preferred for %s." % (siteSpec, d), DEBUG)
            return siteDep
          return d
        deps = [siteSpecific(d) for d in deps]
        sources.extend ([join (cmsdistPath, source + ".file")
                         for source in localSources])
        patches.extend ([join (cmsdistPath, patch + ".patch") 
                        for patch in localPatches])
        self.__requiresCache[self.name] = (deps, sources, patches, buildDeps)
        return (deps, sources, patches, buildDeps)

def parseOptions():
    parser = OptionParser(usage="""
  cmsBuild [<additional-options>] [--no-bootstrap] -a <architecture> -c <cmsdist-dir> build <package> [<package2> .. <packageN>]
  cmsBuild [<additional-options>] [--sync-back] [--no-bootstrap] -a <architecture> -c <cmsdist-dir> upload <package> [<package2> .. <packageN>]""")
    deprecatedOptions = [("--compiling-processes", "--jobs / -j"),
                         ("--workers-pool-size", "--builders")]
    for deprecated, new in deprecatedOptions:
      if deprecated in sys.argv:
        parser.error("Deprecated option name %s. Please use %s." % (deprecated, new))
    
    # FIXME add the relevant options.
    parser.add_option("-c", "--cmsdist", 
                      dest="cmsdist",
                      help="the location of the CMSDIST directory",
                      metavar="PATH",
                      default=abspath("./CMSDIST"))
    parser.add_option("--cmsdist-tag",
                       dest="cmsdistTag",
                       default="",
                       metavar="TAG",
                       help="Tag to fetch for CMSDIST.")
    parser.add_option("--architecture", "-a",
                      dest="architecture",
                      default=None,
                      help="Architecture to be used.")
    parser.add_option("-i", "--work-dir",
                      dest="workDir",
                      default=getcwd(),
                      metavar="PATH",
                      help="the location where the build happens")
    parser.add_option("--jobs", "-j",
                      dest="compilingProcesses",
                      metavar="N",
                      help="The number of processes to be used to compile a given package.",
                      default=1)
    parser.add_option("--repository",
                      dest="repository",
                      metavar="NAME",
                      help="Name of the repository directory on server. E.g. 'cms'.",
                      default="cms")
    parser.add_option("--tag",
                      dest="tag",
                      metavar="NAME",
                      help="Name of the tag. Default is constructed from the --repository <repo>.",
                      default="")
    parser.add_option("--builders",
                      dest="workersPoolSize",
                      metavar="N",
                      default=1,
                      help="Maximum number of packages to be build in parallel.",
                      type="int")
    parser.add_option("--sync-back",
                      dest="syncBack",
                      action="store_true",
                      help="When uploading, copy the temporary directory to the production one.",
                      default=False)
    advancedBuildOptions = OptionGroup(parser, "Advanced build options")
    advancedBuildOptions.add_option("--use-system-compiler",
                                    dest="systemCompiler",
                                    action="store_true",
                                    help="""Use the system compiler rather than the one specified in CMSDIST""",
                                    default=None)
    advancedBuildOptions.add_option("--ignore-compile-errors",
                                    "-k",
                                    dest="ignoreCompileErrors",
                                    help="Ignores CMSSW compilation errors.",
                                    action="store_true",
                                    default=False)
    advancedBuildOptions.add_option("--no-bootstrap",
                                    dest="bootstrap",
                                    action="store_false",
                                    help="Do not bootstrap area and ignore packages on the remote repository.",
                                    default=True)
    advancedBuildOptions.add_option("--skip-pre-install-checks",
                                    action="store_true",
                                    dest="skipPreInstall",
                                    default=None,
                                    help="Skip execution of default CMS pre-install scriplet")
    advancedBuildOptions.add_option("--build-command-prefix",
                                    dest="buildCommandPrefix",
                                    default="",
                                    metavar="COMMAND",
                                    help="String to prepend to any rpmbuild command")
    advancedBuildOptions.add_option("--pretend",
                                    dest="pretend",
                                    action="store_true",
                                    help="Do not actually build",
                                    default=False)
    advancedBuildOptions.add_option("--server",
                                    dest="server",
                                    metavar="URL",
                                    help="URL of apt repository. E.g. 'http://cmsrep.cern.ch/cmssw'",
                                    default="http://cmsrep.cern.ch/cmssw")
    advancedBuildOptions.add_option("--new-scheduler", dest="newScheduler", action="store_true",
                                    default=False, help="Use the new scheduler to install / build packages")
    # FIXME: change option to --no-deprecate and do the deprecation 
    #        automatically by default.
    #parser.add_option ("--deprecate-local",
    #                   dest="doDeprecate",
    #                   action="store_true",
    #                   default=False)
    uploadGroup = OptionGroup(parser, "Upload Options",
                        "Options to fine-tune the upload to repository process.")
    uploadGroup.add_option("--upload-server",
                           dest="uploadServer",
                           metavar="HOSTNAME",
                           help="The hostname of the server where the repository is physically located.",
                           default="cmsrep.cern.ch")
    uploadGroup.add_option("--upload-user",
                           dest="uploadUser",
                           metavar="USERNAME",
                           help="The username by which connect to the server where the repository is located.",
                           default="cmsbuild")
    uploadGroup.add_option("--upload-port",
                           dest="uploadPort",
                           metavar="<0-65535>",
                           help="The port on which the repository sshd is listening to (default 22).",
                           default="22")
    uploadGroup.add_option("--upload-root-directory",
                           dest="uploadRootDirectory",
                           metavar="PATH",
                           default="/data/cmssw",
                           help="The base directory where the repository is located. E.g. /data/cmssw.")
    uploadGroup.add_option("--upload-tmp-repository",
                           dest="uploadTmpRepository",
                           help="Name of temporary repository to use during upload. Default=cms.<username> (Deleted/recreated for each upload request)",
                           default=getuser())

    uploadGroup.add_option("--server-apt-env",
                           dest="serverAptEnv",
                           default="/data/cmssw/apt/etc/profile.d/init.sh",
                           metavar="FILENAME",
                           help="Server side location of the apt environment script to be sourced.")
    
    debugGroup = OptionGroup(parser, "Debug options")
    debugGroup.add_option("--debug",
                          dest="debug",
                          action="store_true",
                          help="Print out debug information.",
                          default=False)
    debugGroup.add_option("--no-cleanup",
                          dest="noCleanup",
                          default=False,
                          action="store_true",
                          help="Do not automatically clean-up stale files.")
    debugGroup.add_option("--specs-only",
                          dest="specsOnly",
                          help="Do not build. Only write out RPM spec files.",
                          action="store_true",
                          default=False)
    debugGroup.add_option("--trace",
                          dest="trace",
                          action="store_true",
                          help="Show activity of builders and scheduler.",
                          default=False)
    debugGroup.add_option("--progress",
                          dest="progress",
                          action="store_true",
                          help="Show progress",
                          default=False)
    parser.add_option_group(advancedBuildOptions)
    parser.add_option_group(uploadGroup)
    parser.add_option_group(debugGroup)
    opts, args = parser.parse_args (None, None)
    
    # Setup some default values.
    # FIXME: use an OptionParser callback.
    # FIXME: exclusive --cmsdist and --cmsdist-tag options.
    # FIXME: change site to "variant" and extract it from the architecture.
    opts.workDir = abspath(opts.workDir)
    opts.site = ""
    if "://" not in opts.cmsdist:
        opts.cmsdist = abspath (opts.cmsdist)
    
    if opts.cmsdistTag:
       opts.cmsdist = "cvs://?tag=%s" % opts.cmsdistTag 
    
    if not opts.architecture:
      parser.error("Please specify the architecture you want to build for.")

    # Check for weird characters in workdir
    if "/." in opts.workDir:
      parser.error("Hidden directories not allowed as --work-dir path.")
      
    if re.match(".*[\\\+[*$].*", opts.workDir):
      parser.error("Please avoid '\\+[*$' in --work-dir path.")
    
    # If --tag is not provided via command line then tag will always match the
    # repository (minus the various .pre bits, like for comp.pre)
    if not opts.tag:
      opts.tag = opts.repository.split(".")[0]
    
    setLogLevel (opts)
    if len(args) < 2:
      parser.error("Please build or upload action. Use --help to see additional options.")

    opts.installDir = opts.workDir
    if (opts.repository == "cms") or opts.repository.startswith("cms."):
        opts.installDir = "/opt/cmssw"

    opts.tempDirPrefix = "tmp"
    if opts.workDir.split(sep)[1] == opts.tempDirPrefix:
        opts.tempDirPrefix += "X"

    if len(args) < 2:
      fatal("Wrong number of arguments.")
    
    if not args[0] in ["build", "upload", "deprecate-local"]:
      fatal("Unknown command %s. Alternatives are build, upload." % args[0])
    
    if opts.syncBack and args[0] != "upload":
      fatal("--sync-back option is available only when uploading.")

    if "-cache" in opts.repository or "-cache" in opts.uploadTmpRepository:
      fatal("Repository name cannot contain -cache")

    return opts, args
    
def createDirs (architecture, dest="./"):
    dirs = ["SPECS", join ("RPMS", architecture), 
            "SRPMS", join ("BUILD", architecture), "SOURCES", "BUILDROOT", "WEB"]
    for d in dirs:
        try:
            makedirs (join (abspath (dest), d))
        except:
            pass

def prepareSaveSpec (p):
    for directory in [p.specdir, p.sourcedir]:
        log ("Creating directory %s" % directory, DEBUG)
        if not exists (directory): makedirs (directory)
    p.dumpCmsos ()

def saveSpec (p):
    f = file (p.finalSpecFilename (), 'w')
    # Removes the "Requires:"
    spec = re.sub ("Requires:[^+\n]*\n", "\n", re.sub ("BuildRequires:[^+\n]*\n", "\n", p.getFinalSpec ()))
    f.write (spec)
    #Fix for RPM 4.4.: We need to add something at the end of the spec file otherwise
    #any multiline macro in the %post section does not expant properly if it is not followed
    #by any other command/comment.
    f.write ("\n#\n");
    f.close ()

def packageList (pkg):
    pkgListCreator = lambda finalString, pkg: "%s, %s" % (finalString, pkg.name)
    pkgList = reduce (pkgListCreator, pkg.dependencies, "")
    pkgList = reduce (pkgListCreator, [pkg], pkgList)
    return pkgList.strip (",")
    
def fetchLocal (pkg, sourceFilename):
    assert (sourceFilename[0] == "/")
    try:
        if not exists (pkg.sourcedir):
            makedirs (pkg.sourcedir)
        destFilename = join (pkg.sourcedir, 
                             basename(sourceFilename.rsplit(".", 1)[0]))
        log ("Copying %s to %s" % (sourceFilename, destFilename), DEBUG)
        source = open (sourceFilename)
        dest = file (destFilename, 'w')
        dest.write (source.read ())
        dest.close ()
        source.close ()
        return (sourceFilename, True)
    except Exception, e:
        return (sourceFilename, False)

def fetchSourcesNew(pkg, scheduler):
  scheduler.log("Fetching sources for %s" % pkg.name)
  sourcedir = pkg.sourcedir
  downloadables = pkg.sources + pkg.patches
  if not downloadables:
    scheduler.log("No sources to be downloaded found for packages %s." % pkg.name)
    return
  urlRe = re.compile (".*:.*/.*")

  # No need to create the download directory
  # not to dump the cmsos, because it was already
  # done.
  for url in downloadables:
    if urlRe.match(url):
      try:
        success = download(url, sourcedir, pkg.options)
      except MalformedUrl, e:
        return str(e)
    else:
      output, success = fetchLocal(pkg, url)
    if not success:
      return "Unable to download %s" % url
    scheduler.log("Done fetching %s" % url)

def fetchSources (pkg):
    log ("Fetching sources for %s" % pkg.name, DEBUG)
    sourcedir = pkg.sourcedir
    if not sources and not patches:
        log ("No sources to be downloaded found for packages %(name)s" % pkg.__dict__)
    urlRe = re.compile (".*:.*/.*")

    def doDownload (source):
        log (source, DEBUG)
        if urlRe.match (source):
            result = download (source, sourcedir, pkg.options)
            if not result:
                raise UnableToDownload (source)
        else:
            output, success = fetchLocal (pkg, source)
            if not success:
                raise UnableToDownload (source)

    if not exists (pkg.sourcedir):
        log ("Creating directory %s" % sourcedir, DEBUG)
        makedirs (sourcedir)

    pkg.dumpCmsos ()
    if pkg.patches:
        log ("Fetching local patches:", DEBUG)
        for url in pkg.patches : doDownload (url)
        
    if pkg.sources:
        log ("Fetching sources: ", DEBUG)
        for url in pkg.sources : doDownload (url)
    return (pkg.name, True)

def checkIfInstalled (pkg):
    """ Check if a given rpm is already in the db.
    """
    if not pkg.pkgName() in rpm_db_cache:
        return False
    
    if not isDefaultRevision(pkg.pkgRevision):
        revision = rpm_db_cache[pkg.pkgName()]
        if revision >= pkg.pkgRevision:
            log("RPM %s found in RPM db with Revision %s (bigger than pkgRevision = %s)" % ( pkg.pkgName (),revision,pkg.pkgRevision), DEBUG)
            return True
        return False
    log ("RPM %s found in RPM db." % pkg.pkgName (), DEBUG)
    return True

def checkIfOnServer (pkg):
    """ Check if the rpm associated to pkg can be downloaded from the apt
        server.
    """
    log ("Checking if package %s can be downloaded from apt servers" % pkg.pkgName(), DEBUG)
    if tags_cache.cache.has_key(pkg.pkgName()) and isDefaultRevision(pkg.pkgRevision):
        return True
    error, output = getstatusoutput ("apt-cache show %s" % pkg.pkgName())
    if error or not output:
        log ("RPM %s not found in apt repository." % pkg.pkgName (), DEBUG)
        return False
    pkgName = None
    pkgRevision = None
   
    lines = [line for line in output.split("\n") if line]
    assert(lines)
    for line in lines:
        key, value = [l.strip() for l in line.split(":", 1)]
        log ("Parsing apt-cache show. key: %s,i value: %s." % (key,value), DEBUG)
        if key == "Package":
            pkgName = value
        if key == "Version":
            pkgRevision = value.split("-",1)[1]
        if key == "Description":
            break
    if pkg.pkgRevision <= pkgRevision and pkgName == pkg.pkgName ():
        log ("RPM %s found in apt repository with a revision greater than the current one. Accepting package." % pkg.pkgName (), DEBUG)
        return True
    log("""RPM %s is found in the apt repository, but it has a earlier revision (%s)
than the current one(%s). Building current package.""" % (pkg.pkgName(), pkgRevision, pkg.pkgRevision), DEBUG)
    return False

def checkCanInstallRpm (pkg):
    officialRpmLocation = pkg.rpmLocation()
    uniqueRpmLocation = join (abspath ("RPMS/cache"), pkg.checksum, pkg.rpmfilename)
    if exists (officialRpmLocation):
        return True
    elif exists (uniqueRpmLocation):
        log ("""No rpm found at:
                %(officialRpmLocation)s 
                but original is actually there at:
                %(uniqueRpmLocation)s""" % locals ())
        symlink (uniqueRpmLocation, officialRpmLocation)
        return True
    return False

def getScriptlets(pkg, relocation, original):
    """Gets the scriptlet associated to an rpm.
    """
    command = "rpm -qp --scripts %s" % pkg.rpmLocation()
    error, output = getstatusoutput(command)
    if error:
        log ("Command: %(command)s failed with the following message: %(output)s." % locals(), DEBUG)
        raise RpmInstallFailed (pkg, output)
    capture = None
    scripts = {}
    secEXP = ""
    for sec in "pre", "post", "postun", "preun":
        scripts[sec] = 'RPM_INSTALL_PREFIX="%s"; export RPM_INSTALL_PREFIX\n' % relocation
        secEXP += sec+"|"
    secPattern = re.compile("^("+secEXP.strip("|")+")install\s+scriptlet\s+")
    for line in output.split("\n"):
        m = secPattern.match(line)
        if m:
            capture = m.group(1)
        elif capture:
            scripts[capture] += line+"\n"
    return scripts

def installRpm (pkg, bootstrap):
    """ Installs an rpm
    """
    error = 1; output = ""; command = "#No command"
    if bootstrap:
        command = "rpm -Uvh --prefix "+pkg.options.workDir+" %s" % pkg.rpmLocation()
        error, output = getstatusoutput (command)
    else:
        workDir = pkg.workDir
        tmpDir  = join(workDir, pkg.tempDirPrefix, pkg.checksum)
        tmpInstall  = join(tmpDir, pkg.installDir.strip(sep))
        rpmLoc  = pkg.rpmLocation()
        command  = "mkdir -p %(tmpInstall)s; rm -rf %(tmpInstall)s; ln -s %(workDir)s %(tmpInstall)s;"
        command += "cd %(tmpDir)s; rpm2cpio %(rpmLoc)s | cpio -idmv; cd %(workDir)s; rm -rf %(tmpDir)s"
        command = command % locals()
        error, output = getstatusoutput(command)
        if not error:
            scripts = getScriptlets(pkg, pkg.options.workDir, "")
            log ("Scriptlets: %s" % scripts, DEBUG)
            postScript = NamedTemporaryFile()
            postScript.write (scripts["post"])
            postScript.flush()
            error, output = getstatusoutput("cat %s; sh -e -x %s" % (postScript.name, postScript.name))
    if error:
        log ("Command:\n %(command)s failed with the following message: %(output)s" % locals (), DEBUG)
        raise RpmInstallFailed (pkg, output)
    log ("Done installing via rpm.", DEBUG)

def installApt (pkg, scheduler):
    command = "apt-get -y install %s" % pkg.pkgName ()
    error, output = getstatusoutput (command)
    log ("About to install %s using apt..." % pkg.pkgName (), DEBUG)
    if error:
        log ("Command:\n %(command)s failed with the following message: %(output)s" % locals (), DEBUG)
        raise RpmInstallFailed (pkg, output)
    log ("Done installing via apt.", DEBUG)
    for dep in pkg.dependencies:
      scheduler.forceDone("check-%s" % dep.pkgName())
      scheduler.forceDone("install-%s" % dep.pkgName())

def scheduleInstallPackage(pkg, scheduler):
  if pkg.name == "system-compiler":
    return False

  bootstrapped = pkg.options.bootstrap
  
  instDir = join(pkg.options.workDir, pkg.pkgrel)
  if not bootstrapped and exists(join(instDir, ".package-checksum")):
    scheduler.log("Package already built into %s. Not building." % instDir)
    # subpackages were already built as part of the main package, but the RPMs still need to be symlinked
    return checkCanInstallRpm(pkg)
  
  if not bootstrapped:
    return False

  scheduler.log("Checking if %s is cached." % pkg.name)
  # FIXME: this is evaluated twice!! look at rpmBuild
  actionName = "install-%s" % pkg.pkgName()
  if checkIfInstalled(pkg):
    scheduler.log("%s already in rpm database. Not building." % pkg.name)
    scheduler.serial(actionName, [], installDoneDummy)
    return True
  if checkCanInstallRpm(pkg):
    scheduler.log("%s is available from local RPMS area. Not building. Installing..." % pkg.pkgName())
    scheduler.serial(actionName, [], installRpm, pkg, bootstrapped)
    return True
  if checkIfOnServer (pkg):
    scheduler.log("%s is available from remote apt-get repository \"%s\". Not building. Downloading and installing..." % (pkg.name, pkg.options.repository))
    aptPackageInstallDeps = []
    for dep in pkg.dependencies + [pkg]:
       if not checkIfInstalled(dep):
         aptPackageInstallDeps.append("downloadrpm-%s" % dep.pkgName())
         scheduler.parallel(aptPackageInstallDeps[-1], [], downloadRpmsForApt, dep)
    scheduler.serial(actionName, aptPackageInstallDeps, installApt, pkg, scheduler)
    return True
  return False
  
def downloadRpmsForApt(pkg):
  src = "%s/%s/RPMS/%s" % (pkg.options.server.rstrip ("/"), pkg.options.repository, pkg.rpmfilename)
  des= "%s/%s/var/lib/cache/%s" % (pkg.options.workDir, pkg.cmsplatf,pkg.cmsplatf)
  downloadUrllib2(src,des,pkg.options)

def getCommandPrefix (options):
    linux32RE = "slc[^_]+_ia32_gcc[^_]+"
    returnString=""
    if re.match (linux32RE, options.architecture):
        returnString = "linux32 %s" % options.buildCommandPrefix
    elif options.buildCommandPrefix:
        returnString = options.buildCommandPrefix
    return returnString.strip ()

def handleStaleFiles (files, noCleanup):
    for staleFile in [f for f in files if exists(f)]:
        if noCleanup:
             raise UnexpectedFile (staleFile)
        log ("Removing stale file " + staleFile, DEBUG)
        unlink (staleFile)

def buildPackage(pkg, scheduler):
  """ This helper function is responsible for building/installing a given spec and it's associated rpm.
      What it does is the following (TODO):
      * Checks if the rpm is already installed in the RPM DB. If yes, exits.
      * Checks if the rpm is already available in the local area. If yes, installs it.
      * Checks if the rpm is already available on the apt server. If yes, downloads, installs and returns.
      If any of the above is true it:
      * Builds the rpm, if it fails, it aborts.
      * Installs the rpm in the local db, if it fails, it aborts."""
  if pkg.name == "system-compiler":
    return
  
  # if this is a subpackge do not try to build it, it should have been built as part of its parent
  if pkg.parent:
    scheduler.log("Not building Subpackage %s, it should have been built as part of Package %s." % (pkg.name, pkg.parent.name))
    return

  scheduler.log("Creating directory %s if not existing." % pkg.builddir)
  if not exists (pkg.builddir):
    makedirs (pkg.builddir)
  logfile = "%s/log" % pkg.builddir
  
  scheduler.log("Building %s. Log can be found in %s." % (pkg.name, logfile))
  optionsDict = {
    "topdir": abspath("."),
    "specdir": pkg.specdir,
    "builddir": pkg.builddir,
    "makeprocesses": "",
    "ignoreCompileErrors": "",
    "nodeps": ""
  }

  rpmsCacheDir = dirname(join(abspath ("RPMS/cache/%s" % pkg.checksum), pkg.rpmfilename))
  finalRpmCacheDir = dirname(join (abspath ("RPMS"), pkg.rpmfilename))
  srpmsCacheDir = join(abspath("SRPMS/cache/%s" % pkg.checksum))
  
  missingDirs = [d for d in [rpmsCacheDir, finalRpmCacheDir, srpmsCacheDir] 
                 if not exists(d)]
  for directory in missingDirs:
    makedirs(directory) 
  
  finalRpmFilename = join(abspath("RPMS"), pkg.rpmfilename)
  uniqueRpmFilename = join(abspath("RPMS/cache/%s" % pkg.checksum), pkg.rpmfilename)
  handleStaleFiles ([finalRpmFilename, uniqueRpmFilename], pkg.options.noCleanup)    
  
  if pkg.options.compilingProcesses:
    optionsDict["makeprocesses"] = "--define \"compiling_processes %s\"" % pkg.options.compilingProcesses
  if pkg.options.ignoreCompileErrors:
    optionsDict["ignoreCompileErrors"] = "--define \"ignore_compile_errors 1\"" 
  optionsDict["tmpdir"] = join(opts.workDir, opts.tempDirPrefix)
  optionsDict.update({"prefix": getCommandPrefix(pkg.options),
                      "buildRoot": join(optionsDict["tmpdir"], "BUILDROOT", pkg.checksum),
                      "extraRpmDefines": pkg.options.rpmQueryDefines})
  if not pkg.options.bootstrap:
    optionsDict["nodeps"] = "--nodeps"

  # FIXME: should be done only once!
  err, out = getstatusoutput("rpmbuild --version")
  if err:
    return "ERROR: unable to find working rpmbuild"
 
  rpmbuildCommand = "TMPDIR=%(tmpdir)s %(prefix)s rpmbuild %(nodeps)s --buildroot %(buildRoot)s -ba --define '_topdir %(topdir)s'" \
                    " %(makeprocesses)s %(ignoreCompileErrors)s %(extraRpmDefines)s" \
                    " %(specdir)s/spec >%(builddir)s/log 2>&1" % optionsDict
  rpmbuildCommand = rpmbuildCommand.strip()
  scheduler.log(rpmbuildCommand)
  error, output = getstatusoutput(rpmbuildCommand)
  if error:
    if not exists(logfile):
      return "Error happened before any log output."
    logLines = open(logfile).readlines()
    return "Failed to build %s. Log file in %s. Final lines of the log file:\n%s" % (pkg.name, logfile, "".join(logLines[-20:]))
  if output:
    return output
  scheduler.log("Build successful.")
  
  # link the RPM from the unique name to the friendly name
  symlink(uniqueRpmFilename, finalRpmFilename)
  # link subpackages, if any
  for subpackage in pkg.subpackages:
    finalSubFilename = join(abspath ("RPMS"), subpackage.rpmfilename)
    uniqueSubFilename = join(abspath ("RPMS/cache/%s" % subpackage.checksum), subpackage.rpmfilename)
    symlink(uniqueSubFilename, finalSubFilename)
  
def installPackage(pkg, scheduler):
  if not checkCanInstallRpm (pkg):
    raise RpmBuildFailed (pkg)
  scheduler.log("Trying to install the rpm package %s just built." % pkg.pkgName ())
  installRpm(pkg, pkg.options.bootstrap)
  scheduler.log("Done")
  #print scheduler.jobs
  f = open(join(pkg.options.workDir, pkg.pkgrel, ".package-checksum"), 'w')
  f.write (pkg.checksum)
  f.close ()
  scheduler.reschedule()

def rpmBuild (pkg, tracker):
    """ This helper function is responsible for building/installing a given spec and it's associated
        rpm. What it does is the following (TODO):
        * Checks if the rpm is already installed in the RPM DB. If yes, exists.
        * Checks if the rpm is already available in the local area. If yes, installs it.
        * Checks if the rpm is already available on the apt server. If yes, downloads, installs and returns.
        If any of the above is true it:
        * Builds the rpm, if it fails, it aborts.
        * Installs the rpm in the local db, if it fails, it aborts."""
    if pkg.name == "system-compiler":
        return
    
    log ("Creating directory %s if not existing." % pkg.builddir, DEBUG)
    if not exists (pkg.builddir):
        makedirs (pkg.builddir)
    logfile = "%s/log" % pkg.builddir
    
    log ("Building %s. Log can be found in %s." % (pkg.name, logfile), DEBUG)
    tracker.postMessage ("BUILD_PKG", {"pkgName": pkg.pkgName (),
                                       "logFile": logfile})
    optionsDict = {
        "topdir": abspath("."),
        "specdir": pkg.specdir,
        "builddir": pkg.builddir,
        "makeprocesses": "",
        "ignoreCompileErrors": ""
    }

    rpmsCacheDir = dirname (join (abspath ("RPMS/cache/%s" % pkg.checksum), pkg.rpmfilename))
    finalRpmCacheDir = dirname (join (abspath ("RPMS"), pkg.rpmfilename))
    srpmsCacheDir = join (abspath ("SRPMS/cache/%s" % pkg.checksum))
    
    for directory in [rpmsCacheDir, finalRpmCacheDir, srpmsCacheDir]:
         if not exists (directory): makedirs (directory) 
    
    finalRpmFilename = join (abspath ("RPMS"), pkg.rpmfilename)
    uniqueRpmFilename = join (abspath ("RPMS/cache/%s" % pkg.checksum), pkg.rpmfilename)

    handleStaleFiles ([finalRpmFilename, uniqueRpmFilename], pkg.options.noCleanup)    
    
    if pkg.options.compilingProcesses:
        optionsDict["makeprocesses"] = "--define \"compiling_processes %s\"" % pkg.options.compilingProcesses
    if pkg.options.ignoreCompileErrors:
        optionsDict["ignoreCompileErrors"] = "--define \"ignore_compile_errors 1\"" 
    
    optionsDict["prefix"] = getCommandPrefix (pkg.options)
    optionsDict["tmpdir"]    = join (opts.workDir, opts.tempDirPrefix)
    optionsDict["buildRoot"] = join (optionsDict["tmpdir"], "BUILDROOT", pkg.checksum )
    optionsDict["extraRpmDefines"] = pkg.options.rpmQueryDefines

    err, out = getstatusoutput("rpmbuild --version")
    if err:
      fatal("ERROR: unable to find working rpmbuild")
 
    rpmbuildCommand = "TMPDIR=%(tmpdir)s %(prefix)s rpmbuild --buildroot %(buildRoot)s -ba --define '_topdir %(topdir)s'" \
                      " %(makeprocesses)s %(ignoreCompileErrors)s %(extraRpmDefines)s" \
                      " %(specdir)s/spec >%(builddir)s/log 2>&1" % optionsDict
    rpmbuildCommand = rpmbuildCommand.strip ()
    log (rpmbuildCommand, DEBUG)
    error, output = getstatusoutput (rpmbuildCommand)
    if error:
        raise RpmBuildFailed (pkg)
    if output:
        log ("WARNING! Unexpected output:")
        log ("%s" % output)
    log ("Build successful.", DEBUG)
   
    # link the RPM from the unique name to the friendly name
    symlink (uniqueRpmFilename, finalRpmFilename)
    # link subpackages, if any
    for subpackage in pkg.subpackages:
        finalSubFilename = join (abspath ("RPMS"), subpackage.rpmfilename)
        uniqueSubFilename = join (abspath ("RPMS/cache/%s" % subpackage.checksum), subpackage.rpmfilename)
        symlink (uniqueSubFilename, finalSubFilename)
    
    if not checkCanInstallRpm (pkg):
        raise RpmBuildFailed (pkg)
    log ("Trying to install the rpm package just built.", DEBUG)
    log ("Acquiring lock", TRACE)
    tracker.globalLock.acquire ()
    log ("Lock acquired", TRACE)
    installRpm (pkg, pkg.options.bootstrap)
    tracker.globalLock.release ()
    log ("Done", DEBUG)
    f = open (join (pkg.options.workDir, pkg.pkgrel, ".package-checksum"), 'w')
    f.write (pkg.checksum)
    f.close ()

from time import sleep

class ActionFactory (object):
    def __init__ (self, pkg):
        self.__package = pkg
        self.__actionList = []
        
    def create (self, cls, parent=None, prevAlternative=None):
        """ Creates an action of kind cls with the specified parent and/or
            alternatives.
        """
        obj  = cls (self.__package, parent, prevAlternative)
        self.__lastCreated = obj
        self.__actionList.append (obj)
        return obj

    def createAlternative (self, cls, prevAlternative=None):
        """ Creates an alternative to prevAlternative action. By default
            prevAlternative is the last action created.
        """
        return self.create (cls, parent=None, 
                            prevAlternative=(prevAlternative or self.__lastCreated))
    
    def createChild (self, cls, parent=None):
        """ Creates a child of the parent action. By default the parent is the
            last action created.
        """
        return self.create (cls, parent=(parent or self.__lastCreated),
                            prevAlternative=None)
                            
    def getActionList (self):
        return self.__actionList
    

def createBuildActionLists (pkg):
    """ Builds the action tree depth first.
    """
    log ("Creating actions for %s" % pkg.pkgName (), DEBUG)
    factory = ActionFactory (pkg)
    factory.create (BuildSystemCompiler)
    factory.createAlternative (InstallFromServer)
    if pkg.sources:
        factory.createAlternative (SourcesDownload)
        factory.createChild (BuildPackage)
    else:
        factory.createAlternative (BuildPackage)
    factory.createChild (LinkPackageFromCache)
    factory.createChild (InstallFromLocalArea)
    return factory.getActionList ()

class BuildChecker (object):
    def __init__ (self, pkg):
        """ This class is used to perform various checks on the build procedure
            and its outcome.
        """
        self.__packages = [pkg]
        self.__actions = []
        for package in self.__packages:
            self.__actions.extend (createBuildActionLists (package))
        for action in self.__actions:
            action.dryRun ()
    
    def checkConsistency (self):
        def checkActionConsistency (action):
            actionName = action.actionName
            pkgName = action.pkg.pkgName ()
            checkSentence = "Checking if possible action %(actionName)s %(pkgName)s behaves as expect..." % locals ()
            willRun = action.dryRun ()
            expectedResults = action.expectedResults ()
            if willRun and not expectedResults:
                log (checkSentence + " NO!!! The following happened:")
                log (action.missingExecution ())
                return False
            if not willRun and expectedResults:
                log (checkSentence + " NO!!! The following happened:")
                log (action.nonExpectedExecution ())
                return False
            log (checkSentence + "YES! %s %s %s and %s" % (actionName, pkgName,
                                                        willRun and "should have run" or "should have not run",
                                                        expectedResults and "looks like it actually was." or "does not look like it was."), DEBUG)
            return True
        message = "Something wrong with action '%s' for package '%s'."
        errorMessage = "\n".join ([message % (action.actionName, action.pkg.pkgName ())
                                   for action in self.__actions
                                   if not checkActionConsistency (action) ])
        if errorMessage:
            log (errorMessage)
            return False
        return True

    def checkIfUploadable (self):
        log ("Checking for anything ready to upload.", DEBUG)
        result = False
        for action in self.__actions:
            if action.dryRun () and action.producesStuffToUpload ():
                log ("Action '%s' for package '%s' produces something to be uploaded." % (action.actionName, 
                                                                                          action.pkg.pkgName ()), DEBUG)
                result = result or True
        return result

# If a package is in apt cache, queue the action to install it.
# If a package is not found in cache, queue actions to download
# sources, build, and install the resulting package.
# Notice that in both cases the action completes successfully.
def checkPackageInCache(pkg, scheduler):
  # make sure subpackages are properly up-to-date with the definitions in their
  # parents
  if pkg.parent:
    pkg.updateFromParent()

  pkgName = pkg.pkgName()
  scheduler.log("Starting to process package %s" % pkgName)
  sourcedir = pkg.sourcedir
  if not exists (sourcedir):
    log ("Creating directory %s" % sourcedir, DEBUG)
    makedirs (sourcedir)
    pkg.dumpCmsos ()
  # Check if it possible to install package and enqueue 
  # serial task to do it.
  # FIXME: Clean up the fact queuing of tasks is actually
  #       a consequence of invoking scheduleInstallPackage.
  scheduled = scheduleInstallPackage(pkg, scheduler)
  if scheduled:
    scheduler.log("Package %s found in repository" % pkgName)
    scheduler.reschedule()
    return
  scheduler.log("Package %s not found in repository. Queuing for build." % pkgName)
  fetchDependencies = []
  if pkg.name.endswith("-patch"):
    baseTool = pkg.name[:-6]
    baseToolVersion = [d.version for d in pkg.dependencies if d.name == baseTool]
    if len(baseToolVersion)>0:
      baseToolVersion = "cms+%s+%s" % (baseTool, baseToolVersion[0])
      if not rpm_db_cache.has_key(baseToolVersion): fetchDependencies = ["install-%s" % baseToolVersion]
  scheduler.parallel("fetch-%s" % pkgName, fetchDependencies, fetchSourcesNew, pkg, scheduler)
  buildActionDependencies = []
  for p in pkg.origDependencies + pkg.origBuildDependencies:
    if not isPackageInstalled(p):
      scheduler.serial("check-%s" % p.pkgName(), [], checkPackageInCache, p, scheduler)
      buildActionDependencies.append("install-%s" % p.pkgName())
  buildActionDependencies.append("fetch-%s" % pkgName)
  installActionDependencies = ["build-%s" % pkgName]
  scheduler.log("Dependencies for %s: %s" % (pkgName, buildActionDependencies))
  scheduler.parallel("build-%s" % pkgName, buildActionDependencies, buildPackage, pkg, scheduler)
  scheduler.serial("install-%s" % pkgName, installActionDependencies, installPackage, pkg, scheduler)
  #Install sub-packages too if any
  subinstallActionDependencies = ["install-%s" % pkgName]
  for subpkg in pkg.subpackages:
    scheduler.serial("install-%s" % subpkg.pkgName(), subinstallActionDependencies, installPackage, subpkg, scheduler)
  scheduler.reschedule()

# Helper to find an unique list 
def uniq(input):
  output = []
  for x in input:
    if x not in output:
      output.append(x)
  return output

# New version using the new scheduler.
def buildSpecsNew(topLevelPackages):
  from scheduler import Scheduler
  scheduler = Scheduler(topLevelPackages[0].options.workersPoolSize)
  for pkg in topLevelPackages:
    if not isPackageInstalled(pkg):
      scheduler.serial("check-%s" % pkg.pkgName(), [], checkPackageInCache, pkg, scheduler)

  # We create before hand since we know they are going to be needed. This
  # is to avoid having parallel threads creating them at the same time with
  # one of the two failing because the directory is already there.
  # FIXME: clean up the other places where these are created but only
  #        after we move to the new scheduler.
  for path in ["SOURCES/cache", "RPMS", "SRPMS", "BUILD", "SPECS", "BUILDROOT"]:
    d = join(pkg.options.workDir, path)
    if not exists(d):
      makedirs(d)

  scheduler.run()
  for (action, error) in scheduler.errors.iteritems():
    log("* The action \"%s\" was not completed successfully because %s" % (action, error))
  if scheduler.brokenJobs:
    sys.exit(1)

def build(opts, args, factory):
  err, out = getstatusoutput("rpm --version")
  if err:
    fatal("unable to find a working rpm.")
  packages = [factory.createWithSpec(pkgName) for pkgName in args]
  packages.sort()
  for pkg in packages:
    logMessage = "Package %s requested." % pkg.pkgName ()
    if not isDefaultRevision(pkg.pkgRevision):
      logMessage += " Forcing it at revision %s." % pkg.pkgRevision
    log (logMessage)
    if pkg.fullDependencies:
      log ("This will bring in also the following packages: ")
      deps = sorted(pkg.fullDependencies, key = lambda d: d.pkgName().lower())
      finalString = "\n".join (["%s (%s)" % (dep.pkgName (), dep.checksum) 
                                for dep in deps])
      log (finalString)
  
  if opts.specsOnly:
      log ("SPECS files written")
      sys.exit (0)
  
  if opts.pretend:
      log ("Option --pretend specified. Not building.")
      return
  
  rpmCacheUpdate(opts)
  buildSpecsNew(packages)

def isPackageInstalled(pkg):
    if pkg.options.bootstrap:
        pkgName = pkg.pkgName ()
        if rpm_db_cache.has_key(pkgName):
            if not isDefaultRevision(pkg.pkgRevision):
                if rpm_db_cache[pkgName] < pkg.pkgRevision:
                    return False
            log ("Package %s with same/newer revision %s is already installed" % (pkgName, str(rpm_db_cache[pkgName])), DEBUG)
            return True
    else:
        instDir = join(pkg.options.workDir, pkg.pkgrel)
        if exists(join(instDir, ".package-checksum")):
            return True
    return False

def fetch (opts, args, factory):
    pkgList = []
    for pkgName in args:
        # do not care about subpackages, they come from the same sources as the master
        pkg = factory.createWithSpec(pkgName)
        createDirs(architecture=pkg.cmsplatf)
        pkgList += pkg.fullDependencies + [pkg]
    for pkg in pkgList:
        fetchSources (pkg)

def sources (opts, args, factory):
    for pkgName in args:
        # do not care about subpackages, they come from the same sources as the master
        pkg = factory.createWithSpec (pkgName)
        log ("The following files are going to be fetched for package %s." % pkg.name)
        [log ("Source: %s" % source) for source in pkg.sources if source != "none"]
        [log ("Patch: %s" % source) for source in pkg.patches if source != "none"]

class PlatformDetectionError (Exception):
    pass

def callCmsos (opts):
    pathname = join (opts.cmsdist, "cmsos.file")
    commandPrefix = getCommandPrefix (opts)
    error, output = getstatusoutput (("%s sh %s" % (commandPrefix, pathname)).strip ())
    if error:
        raise PlatformDetectionError ("Error while executing cmsos.file:\n%s" % output)
    cmsos = output.strip ("\n")
    log ("cmsos value is %s" % cmsos, DEBUG)
    return cmsos

# The helper method responsible for bootstrapping an area.
# @a opts the build options passed on command line.
# @return the location of the init.sh script to be sourced when building 
#         or uploading.
def bootstrap(opts):
  bootstrapUrl = "%s/%s/bootstrap.sh" % (opts.server, opts.repository)
  try:
    f=urlopen(bootstrapUrl)
    bootstrapContents = f.read()
  except:
    log("Unable to fetch file %s. Mispelled architecture name?\nNothing changed, aborting bootstrap." % bootstrapUrl)
    sys.exit (1)
  bootstrapFilename = join(opts.tempdir, "bootstrap.sh")
  bootstrapFile = open(bootstrapFilename, "w")
  bootstrapFile.write(bootstrapContents)
  bootstrapFile.close()
      
  log("Bootstrapping from server %s" % opts.server, DEBUG)
  boostrapServer = opts.server.replace("http://","").replace("https://","")
  parts = boostrapServer.split("/", 1) 
  if len(parts) == 2:
    bootstrapServer = parts[0]
    bootstrapServerPath = "-server-path " + parts[1]
  else:
    bootstrapServerPath = ""
  options = ["-server", bootstrapServer, 
             bootstrapServerPath,
             "-arch",  opts.architecture,
             "-path", opts.workDir,
             "-repository", opts.repository,
             "-assume-yes", "-only-once", "setup"]
  if logLevel is DEBUG:
    options.append("-debug")
  bootstrapCommand ="TMPDIR=%s sh -ex %s %s" % (opts.tempdir, bootstrapFilename, " ".join(options)) 
  log("Bootstrapping cmsBuild area.")
  log(bootstrapCommand, DEBUG)
  error, output = getstatusoutput(bootstrapCommand)
  bootstrapLog = join(opts.tempdir, "bootstrap.log")
  f = open(bootstrapLog, "w")
  f.write(output)
  f.close()
  if error:
    fatal("unsuccessful bootstrap, log can be found in %s." % bootstrapLog)

  parts = glob(join(opts.workDir,opts.architecture, "external/apt/*/etc/profile.d/init.sh"))
  if len(parts) != 1:
    fatal("""malformed bootstrap. More than one apt installation found.""")

  initSh = parts[0]

  if not exists(initSh):
    fatal("""ERROR: cannot read apt-get environment. File missing: %s.""" % s)

  # If we are not in a cfg file write where to find the correct init.sh
  log("Done. Setup log can be found in %s." % bootstrapLog)
  log("init.sh can be found in %s" % initSh, DEBUG)
  createDirs(architecture=opts.architecture, dest=opts.workDir)
  return initSh

def checkFileOnServer (contactString, filename):
    command = """ssh %(contactString)s if [ -f %(file)s ]; then echo 1; fi"""
    error, result = getstatusoutput (command % locals ()).strip ("1")
    if error or not result:
        return False
    return (result == "1" and True) or False

def upload_help ():
    log ("""cmsBuild upload <package to upload>""")    

def check (opts, args, factory):
    """ This method is responsible for checking that a given package was build
        and installed correctly.
    """
    pkgs = [ factory.createWithSpec (specName) for specName in args ]
    # expand subpackages to be checked
    pkgs = factory.expandSubpackages(pkgs)
    unconsistentPkgs = [ pkg
                         for pkg in pkgs
                         if not BuildChecker (pkg).checkConsistency () ]
    if unconsistentPkgs:
        log ("Warning the following packages have errors:")
        log ("\n".join ([pkg.name for pkg in unconsistentPkgs]))
        sys.exit (1)
    log ("Everything ok.")    

def format(s, **kwds):
  return s % kwds

def statusAndLog(status, s, **kwds):
  log(s % kwds)
  return status

def executeScript(opts, dumpedScript, local=True):
  if local:
    executionCmd="sh -e %(debug)s %(filename)s"
  else:
    executionCmd="ssh < %(filename)s -p %(cmdPort)s %(cmdUser)s@%(cmdServer)s sh -e %(debug)s"
  # Run (or print-out) the script
  if not opts.pretend:
    fd, filename = mkstemp(".sh", "cmsBuildExecuted", opts.tempDirPrefix, True)
    f = os.fdopen(fd, "w")
  else:
    f = sys.stderr
  f.write(dumpedScript)
  f.close()
  if not opts.pretend:
    script = format(executionCmd,
                    debug=opts.debug and "-x" or "",
                    filename=filename,
                    cmdServer=opts.uploadServer,
                    cmdUser=opts.uploadUser,
                    cmdPort=opts.uploadPort)
    err, out = getstatusoutput(script)
  if opts.debug:
    log(out)
  return (err, out)

def allPackagesToUpload (args, factory):
  pkgs = [factory.createWithSpec(specName) for specName in args]
  if len(pkgs) !=  len(args):
    return statusAndLog(False, "ERROR: Error while parsing spec.")

  def allDeps(pkg, factory):
    deps = [pkg]
    for dep in pkg.fullDependencies:
      deps += allDeps(dep, factory)
    return deps

  fullPackageList = []
  for pkg in pkgs:
    for dep in allDeps(pkg, factory):
      if dep not in fullPackageList and exists (dep.rpmLocation()):
        fullPackageList.extend(factory.expandSubpackages([dep]))
  return fullPackageList

def  deprecateLocalCommand(opts, fullPackageList):
  if not fullPackageList: return "true"
  deprecablePackages = " ".join([pkg.pkgName() for pkg in fullPackageList])
  deprecableRPMS = " ".join([p.rpmfilename for p in fullPackageList])
  deprecatedSrc  = " ".join(["SOURCES/"+p.pkgdir for p in fullPackageList])
  deprecatedDir  = " ".join([p.pkgrel for p in fullPackageList])
  deprecatedSRPM = " ".join(["SRPMS/cache/*/"+p.pkgName()+"*" for p in fullPackageList])
  deprecateLocal = format("rpm -e %(deprecablePackages)s\n"
                          'cd %(workDir)s\n'
                          'rm -rf %(deprecatedDir)s\n'
                          "for x in %(deprecableRPMS)s; do\n"
                          '  [ -e RPMS/$x ] || continue\n'
                          '  realfile=`readlink RPMS/$x`\n'
                          '  rm -f RPMS/$x\n'
                          '  rm -f $realfile\n'
                          "done\n"
                          "for x in %(deprecatedSrc)s; do\n"
                          '  [ -d $x ] || continue\n'
                          "  for f in $x/*; do\n"
                          '    realfile=`readlink $f || true`\n'
                          '    [ "$realfile" ] || continue \n'
                          '    rm -f $realfile\n'
                          "  done\n"
                          '  rm -rf $x\n'
                          "done\n"
                          "rm -rf %(deprecatedSRPM)s\n",
                          architecture=opts.architecture,
                          deprecableRPMS=deprecableRPMS,
                          deprecatedSrc=deprecatedSrc,
                          workDir=opts.workDir,
                          deprecatedDir=deprecatedDir,
                          deprecatedSRPM=deprecatedSRPM,
                          deprecablePackages=deprecablePackages)
  return deprecateLocal

def upload(opts, args, factory):
  """New upload method. Updates using this are atomic and it should
     automatically retry when parallel updates happen.
     The workflow is the following.

     * If it exists as a real directory, migrate old style repository to
       me in <repository>-cache/<repository>.00..00-00..00.  We add a symlink
       "<repository>" pointing to it, so that we are compatible with old areas.
     * If the repository does not exists, create it from scratch with
       the naming convention used above.
     * Get the final part of the hash of the repository pointed by the symlink called
       "<repository>". In case we start from scratch this will be 00..00, for example.
     * Check consistency of the upload area just like before.
     * Create locally a delta repository containing all the bits that need to
       be uploaded.
     * Upload the local delta repository to a unique temporary directory on server.
     * rsync the parent repository in a temporary "new area", using --link-dest to minimize
       disk-space. Notice we do not use --link-dest for files which will have to be rewritten 
       (e.g. apt md5cache)
     * rsync the uploaded delta in the same temporary new area, ignoring files
       already existing. This will avoid any kind of overwriting of past repositories, which are
       effectively immutable.
     * calculate the new hash of the temporary area by taking a sha256 sum of
       the file listing in RPMS/cache. This listing will include, by construction,
       all the hashes for the packages and will therefore uniquely identify the 
       repository.
     * Move the new temporary ara in it's final location 
       
       <repository>-cache/<repository>.00..00-XX..XX

       where XX..XX is the hash calculated above.
     * Create a unique symlink:
       
       <repository>.next-XXXXXXX
       
       this will uniquely identify this session.
     * Kill all the other sessions by removing all the symlinks called <repository>.next-* apart from
       us.
     * Get the parent hash and check it is still the one when we began.
     * Swap <repository>.next-XXXXXXX and <repository>. This will happen atomically. This will
       effectively commit the transaction and <repository> will now point to
       the updated repository.
     
     If at any time there is an error (e.g. a parallel upload succeeded before us), we start again
     from scratch, including the consistency check.
     If the consistency check fails, we need to rebuild and therefore we die.
  """
  if not args:
    upload_help()
    return False

  # A few helpers
  def getResult(s, k):
    return [x.replace(k,"") for x in s.split("\n") if x.startswith(k)][0]
  
  def logAndDieOnError(err, msg):
    if not err:
      return
    log(msg)
    sys.exit(1)

  migrateRepo = format('CACHE="%(uploadDir)s/%(repository)s-cache"\n'
                       'mkdir -p $CACHE\n'
                       '# Automatically migrate old style repositories by assigning them the 00..00 hash\n'
                       '# Notice that we keep <repository>.<user> and <repository> in two separate\n'
                       '# caches so that temporary uploads do not overlap with production ones and\n'
                       '# make sure migrated old style repositories do not overlap.\n'
                       'if [ -x %(uploadDir)s/%(repository)s ] && [ ! -L %(uploadDir)s/%(repository)s ]; then \n'
                       '  echo "Migrating old style repository"\n'
                       '  mv %(uploadDir)s/%(repository)s $CACHE/%(repository)s.0000000000000000000000000000000000000000000000000000000000000000-0000000000000000000000000000000000000000000000000000000000000000\n'
                       '  ln -sf $CACHE/%(repository)s.0000000000000000000000000000000000000000000000000000000000000000-0000000000000000000000000000000000000000000000000000000000000000 %(uploadDir)s/%(repository)s\n'
                       'fi\n'
                       'PARENTDIR="`readlink %(uploadDir)s/%(repository)s || true`"\n'
                       'if [ "X$PARENTDIR" = X ]; then\n'
                       '  PARENTDIR="$CACHE/%(repository)s.0000000000000000000000000000000000000000000000000000000000000000-0000000000000000000000000000000000000000000000000000000000000000"\n'
                       '  mkdir -p $PARENTDIR/{RPMS,SRPMS,SOURCES,TARS,WEB,apt,md5cache}\n'
                       '  ln -sf $PARENTDIR %(uploadDir)s/%(repository)s\n'
                       'fi\n'
                       'PARENTHASH=`echo $PARENTDIR | sed -e "s|.*-||"`\n'
                       'echo PARENTDIR:$PARENTDIR\n'
                       'echo RESULT:$PARENTHASH',
                       uploadDir=opts.uploadRootDirectory,
                       repository=opts.repository)
  (err, parentHashResult) = executeScript(opts, migrateRepo, False)
  logAndDieOnError(err, "Error while migrating repository structure.")
  parentDirectory = getResult(parentHashResult, "PARENTDIR:")
  parentHash      = getResult(parentHashResult, "RESULT:")

  #Once we have established the parent then we should update the apt cache
  tags_cache.update()
  fullPackageList = allPackagesToUpload (args, factory)
  if not fullPackageList:
    return statusAndLog(True, "Nothing needs to be uploaded")
  cmsplatf = fullPackageList[0].cmsplatf
  uploadablePackages = []
  for pkg in fullPackageList:
    checker = BuildChecker(pkg)
    if not checker.checkConsistency():
      statusAndLog(False, "ERROR: Build area not consistent. Not uploading.")
      executeScript(opts, deprecateLocalCommand(opts, fullPackageList))
      return False
    if not checker.checkIfUploadable():
      statusAndLog(True, "Nothing needs to be uploaded for %(name)s.",
                                name=pkg.pkgName())
      continue
    uploadablePackages.append(pkg)
  fullPackageList = uploadablePackages
  if not fullPackageList:
    return statusAndLog(True, "Nothing needs to be uploaded")
  log("Ready to upload.")

  packageNames = [pkg.pkgName() for pkg in fullPackageList]
  pkgChecksums = [p.checksum for p in fullPackageList]

  tmpdir=join(opts.workDir, opts.tempDirPrefix, "upload")
  subsystems = " ".join(set([name.split("+")[0] for name in packageNames]))
  # Create an area locally which has to be merged with the remote repository.
  createEmptyRepository = format(
    "set -e\n"
    "rm -rf %(tmpdir)s\n"
    "mkdir -p %(tmpdir)s/RPMS/%(architecture)s\n"
    "mkdir -p %(tmpdir)s/RPMS/cache\n"
    "mkdir -p %(tmpdir)s/SRPMS\n"
    "mkdir -p %(tmpdir)s/SOURCES/%(architecture)s\n"
    "mkdir -p %(tmpdir)s/SOURCES/cache\n"
    "mkdir -p %(tmpdir)s/apt/%(architecture)s\n"
    "mkdir -p %(tmpdir)s/TARS\n"
    "mkdir -p %(tmpdir)s/WEB\n"
    'for x in %(subsystems)s; do\n'
    '  mkdir -p %(tmpdir)s/apt/%(architecture)s/RPMS.$x\n'
    'done\n',
    architecture=opts.architecture,
    tmpdir=tmpdir,
    subsystems=subsystems)
  # Hard-link packages.
  hardLinkPackages = format(
    "%(rsync)s -am --include '*/' %(includes)s --exclude '*' --link-dest %(workdir)s/RPMS/cache/ %(workdir)s/RPMS/cache/ %(tmpdir)s/RPMS/cache/\n"
    "%(rsync)s -am --link-dest %(workdir)s/WEB/ %(workdir)s/WEB/ %(tmpdir)s/WEB/\n"
    ,
    rsync="rsync --chmod=a+rX",
    workdir=opts.workDir,
    tmpdir=tmpdir,
    includes=" ".join(["--include \"**%s**\"" % x for x in pkgChecksums]),
    architecture=opts.architecture)
  # Copy bootstrap.sh, bootstrap-driver.txt, cmsos. Given we can rollback now
  # there is no risk of screwing up the repository by copying some unwanted
  # driver / bootstrap.sh / cmsos.
  # FIXME: copy also some predefined web area, so that we have rollbackable web
  #        area?
  copyByProducts = format("cp `ls -rt %(workdir)s/%(architecture)s/external/bootstrap-driver/*/%(architecture)s-driver.txt | tail -1` %(tmpdir)s/ || true\n"
                          "cp `ls -rt %(workdir)s/%(architecture)s/external/bootstrap-driver/*/%(architecture)s-driver-comp.txt | tail -1` %(tmpdir)s/ || true\n"
                          "cp `ls -rt %(workdir)s/%(architecture)s/external/apt/*/bin/bootstrap.sh | tail -1` %(tmpdir)s/ || true\n"
                          "cp `ls -rt %(workdir)s/%(architecture)s/cms/cms-common/*/*/common/cmsos | tail -1` %(tmpdir)s/cmsos || true\n",
                          workdir=opts.workDir,
                          tmpdir=tmpdir,
                          architecture=opts.architecture)
  # Create the needed symlinks.
  symlinks = ""
  for pkg in fullPackageList:
    symlinks += format(
      "ln -s ../../RPMS/cache/%(checksum)s/%(architecture)s/%(packageName)s %(tmpdir)s/RPMS/%(architecture)s/%(packageName)s\n"
      "ln -s ../../../RPMS/cache/%(checksum)s/%(architecture)s/%(packageName)s %(tmpdir)s/apt/%(architecture)s/RPMS.%(subsystem)s/%(packageName)s\n", 
      checksum=pkg.checksum,
      tmpdir=tmpdir,
      architecture=opts.architecture,
      subsystem=pkg.group,
      packageName=basename(pkg.rpmfilename))

  remoteSourceRE=re.compile (".*:.*/.*")
  sources = ""
  for pkg in [pkg for pkg in fullPackageList if exists (pkg.rpmLocation())]:
    # FIXME: this should really parse the url to determine the output filename
    #        rather than relying on the fact we always end up urls with
    #        /some-file.tar.gz
    sourceFiles = [(source.rsplit("/", 1)[1], getUrlChecksum(source))
                   for source in pkg.sources
                   if remoteSourceRE.match(source)]
    for filename, checksum in sourceFiles:
      sources += format(
        "mkdir -p %(tmpdir)s/SOURCES/cache/%(checksum)s\n"
        "mkdir -p %(tmpdir)s/SOURCES/%(architecture)s/%(group)s/%(name)s/%(version)s\n"
        "ln -f %(workdir)s/SOURCES/cache/%(checksum)s/%(filename)s %(tmpdir)s/SOURCES/cache/%(checksum)s/%(filename)s\n"
        "ln -sf ../../../../../SOURCES/cache/%(checksum)s/%(filename)s %(tmpdir)s/SOURCES/%(architecture)s/%(group)s/%(name)s/%(version)s/%(filename)s\n",
        workdir=opts.workDir,
        tmpdir=tmpdir,
        architecture=opts.architecture,
        group=pkg.group,
        name=pkg.name,
        version=pkg.version,
        filename=filename,
        checksum=checksum)
  

  (err, out) = executeScript(opts, "\n".join([createEmptyRepository, hardLinkPackages, copyByProducts, symlinks, sources]))
  logAndDieOnError(err, "Error while creating the local repository.")

  repoInfo = {"uploadDir": opts.uploadRootDirectory,
              "repository": opts.repository,
              "uploadTmpRepository": opts.uploadTmpRepository,
              "rsync": format("rsync -e 'ssh -p %(port)s' --rsync-path=/usr/bin/rsync --chmod=a+rX",port=opts.uploadPort),
              "server": opts.uploadServer,
              "user": opts.uploadUser,
              "parentHash": parentHash}

  # Create unique repository on server
  (err, createUploadTmp) = executeScript(opts, format("mkdir -p  %(uploadDir)s/tmp\n"
                                                "echo RESULT:`mktemp -d -p %(uploadDir)s/tmp/ tmp.%(parentHash)s-XXXXXXXX`", **repoInfo),
                                         False)
  logAndDieOnError(err, "Error while creating temporary directory.")
  tmpUploadDir = getResult(createUploadTmp, "RESULT:")
  
  # Before uploading the rpms to server temp area, make sure no one is using
  # the same parent
  upload = format(
                  'err=0\n'
                  'while pgrep -f %(originalParentHash)s 2>&1 >/dev/null; do\n'
                  '  echo "Someone is already uploading to the same parent. Avoid congestion by sleeping and trying again."\n'
                  '  sleep 30\n'
                  '  err=1\n'
                  'done\n'
                  'if [ $err = 1 ] ; then\n'
                  '  rm -rf %(tmpUploadDir)s\n'
                  '  exit 1\n'
                  'fi\n',
                  tmpdir=tmpdir,
                  tmpUploadDir=tmpUploadDir,
                  originalParentHash=parentHash,
                  **repoInfo)
  (err, msg) = executeScript(opts, upload, False)
  if err:
    log("Parallel upload session in progress. Starting uploading procedure from scratch. This will check again build area consistency.")
    if opts.debug:
      log(msg)
    return False

  # Upload local repository to a temp area on the server.
  upload = format(
                  '%(rsync)s -a %(tmpdir)s/ %(user)s@%(server)s:%(tmpUploadDir)s/',
                  tmpdir=tmpdir,
                  tmpUploadDir=tmpUploadDir,
                  **repoInfo)
  (err, msg) = executeScript(opts, upload, True)
  if err:
    log("Something went wrong while uploading the rpms to server temp area.")
    if opts.debug:
      log(msg)
    return False
  # Human readable repositories are symlinks to one called:
  # <prefix>.<parent>-<me>
  # 
  # where <prefix> is the --repository option passed on command line, e.g. cms.
  #       <parent> is the sha256 of the previous repository listing.
  #       <me> is the sha256 of the current repository listing.
  # Notice that if the repository is not yet a link we use
  #       <prefix>.0000000000000000-<me>
  # where me is the sha256 of the repository listing, and then we symlink it to 
  # <prefix>.
  # If the repository does not exists at all, we create a dummy:
  # <prefix>.00...00-00...00
  # This allows us to have migrations as well, by tweaking the rsync rule to ignore old
  # architectures for example.
  #
  # We then create a
  createRepo = format(
    'CACHE="%(uploadDir)s/%(repository)s-cache"\n'
    'test -d $CACHE\n'
    'PARENTDIR="`readlink %(uploadDir)s/%(repository)s`"\n'
    'PARENTHASH=`echo $PARENTDIR | sed -e "s|.*-||"`\n'
    'UPLOADREPO=%(tmpUploadDir)s\n'
    'if [ ! "X$PARENTHASH" = X%(originalParentHash)s ]; then\n'
    '  DELME=`echo $UPLOADREPO.delme | sed -e "s/$PARENTHASH//g"`\n'
    '  mv $UPLOADREPO $DELME\n'
    '  rm -rf $DELME\n'
    '  echo "Parent mismatch."\n'
    '  exit 1\n'
    'fi\n'
    '# Check if there are already more than 4 rsync running, \n'
    '# and try again in such a case.\n'
    'err=0\n'
    'while pgrep -f $PARENTHASH 2>&1 >/dev/null; do\n'
    '  echo "Someone is already uploading to the same parent. Avoid congestion by sleeping 1 minute and trying again."\n'
    '  sleep 30\n'
    '  err=1\n'
    'done\n'
    'if [ $err = 1 ] ; then\n'
    '  DELME=`echo $UPLOADREPO.delme | sed -e "s/$PARENTHASH//g"`\n'
    '  mv $UPLOADREPO $DELME\n'
    '  rm -rf $DELME\n'
    '  exit 1\n'
    'fi\n'
    'TMPREPO=%(tmpUploadDir)s-target\n'
    'mkdir -p $TMPREPO/{RPMS,SRPMS,SOURCES,TARS,WEB,apt,md5cache}\n'
    '# First copy / link the old repository.\n'
    '%(rsync)s -a --link-dest $PARENTDIR/RPMS/ $PARENTDIR/RPMS/ $TMPREPO/RPMS/\n'
    '%(rsync)s -a --link-dest $PARENTDIR/SRPMS/ $PARENTDIR/SRPMS/ $TMPREPO/SRPMS/ || true\n'
    '%(rsync)s -a --link-dest $PARENTDIR/SOURCES/ $PARENTDIR/SOURCES/ $TMPREPO/SOURCES/\n'
    '%(rsync)s -a --link-dest $PARENTDIR/TARS/ $PARENTDIR/TARS/ $TMPREPO/TARS/ || true\n'
    '%(rsync)s -a --link-dest $PARENTDIR/WEB/ $PARENTDIR/WEB/ $TMPREPO/WEB/ || true\n'
    '%(rsync)s -a $PARENTDIR/apt/ $TMPREPO/apt/\n'
    '%(rsync)s -a $PARENTDIR/md5cache/ $TMPREPO/md5cache/\n'
    '# Copy ancillary files from $UPLOADEDREPO first, so that newer builds override old.\n'
    '%(rsync)s -a $PARENTDIR/{bootstrap.sh,*-driver.txt,cmsos} $TMPREPO/ || true\n'
    '%(rsync)s -a --ignore-existing $UPLOADREPO/{bootstrap.sh,*-driver.txt,cmsos} $TMPREPO/ || true\n'
    '# Then copy the new one, ignoring existing files. This way we make sure no files can be overwritten.\n'
    '%(rsync)s -a --ignore-existing $UPLOADREPO/RPMS/ $TMPREPO/RPMS/\n'
    '%(rsync)s -a --ignore-existing $UPLOADREPO/SRPMS/ $TMPREPO/SRPMS/\n'
    '%(rsync)s -a --ignore-existing $UPLOADREPO/SOURCES/ $TMPREPO/SOURCES/\n'
    '%(rsync)s -a --ignore-existing $UPLOADREPO/TARS/ $TMPREPO/TARS/\n'
    '%(rsync)s -a --ignore-existing $UPLOADREPO/apt/ $TMPREPO/apt/\n'
    '# Web area will always be the latest one built,\n'
    '# however we still hardlink old, unchanged, content.\n'
    '%(rsync)s -a --link-dest $PARENTDIR/WEB/ $UPLOADREPO/WEB/ $TMPREPO/WEB/\n'
    'CHILDHASH=`find $UPLOADREPO/RPMS/cache -type f -name "*.rpm" | sed -e "s|^.*/RPMS/cache||" | sort | sha256sum | cut -f1 -d\ `\n'
    'rm -rf $UPLOADREPO\n'
    '# Generate a new apt cache.\n'
    'source %(serverAptEnv)s\n'
    'PREVIOUSSUBSYS=`ls $PARENTDIR/apt/%(cmsplatf)s/base/pkglist.*.bz2 | sed -e "s|.*[.]\\(.*\\)[.]bz2|\\1|"`\n'
    'for x in %(subsystems)s $PREVIOUSSUBSYS; do\n'
    '  mkdir -p $TMPREPO/apt/%(cmsplatf)s/RPMS.$x\n'
    'done\n'
    'for x in `find $TMPREPO/md5cache/%(cmsplatf)s -name "*.md5cache"`; do\n'
    '  mv $x `dirname $x`/`echo $TMPREPO | tr / _`_apt_%(cmsplatf)s`echo $x | sed -e"s/.*%(cmsplatf)s//"`\n'
    'done\n'
    'genbasedir --cachedir=$TMPREPO/md5cache/%(cmsplatf)s --flat $TMPREPO/apt/%(cmsplatf)s `echo %(subsystems)s $PREVIOUSSUBSYS | tr " " "\n" | sort -u`\n'
    'chmod 755 $TMPREPO/apt/%(cmsplatf)s\n'
    '# Move the new repository into place and link it to its <repo>.<user> name.\n'
    'CHILD="$CACHE/%(repository)s.${PARENTHASH}-${CHILDHASH}"\n'
    'if [ ! -e $CHILD ] ; then\n'
    '  mv -T $TMPREPO $CHILD || { mv $TMPREPO $TMPREPO.delme && exit 1; }\n'
    'else\n'
    '  mv $TMPREPO $TMPREPO.delme\n'
    'fi\n'
    'if [ %(syncBack)s = True ]; then\n'
    '  TARGETREPO=%(repository)s\n'
    'else\n'
    '  TARGETREPO=%(repository)s.%(uploadTmpRepository)s\n'
    '  if [ -x %(uploadDir)s/$TARGETREPO ] && [ ! -L %(uploadDir)s/$TARGETREPO ] ; then\n'
    '    mv %(uploadDir)s/$TARGETREPO %(uploadDir)s/delme.$TARGETREPO\n'
    '  fi\n'
    'fi\n'
    'UNIQUEDIR=`mktemp -d -t cmsBuildUpload.XXXXXXXXXX`\n'
    'UNIQUEID=`echo $UNIQUEDIR | sed -e "s|.*cmsBuildUpload[.]||"`\n'
    'echo $UNIQUEID\n'
    '# Create a transaction.\n'
    'ln -sf $CHILD %(uploadDir)s/$TARGETREPO.next-$UNIQUEID\n'
    '# Try to abort all the pending transactions but ours.\n'
    'for x in `find %(uploadDir)s/$TARGETREPO.next-* ! -name "*-$UNIQUEID"`; do\n'
    ' if rm $x; then true; else rm -f %(uploadDir)s/$TARGETREPO.next-$UNIQUEID ; fi\n'
    'done\n'
    '# In case no transaction is aborted either parallel transactions completed\n'
    '# successfully (and the parenthash is not the same anymore) or there\n'
    '# were no other transactions.\n'
    'PARENTDIR="`readlink %(uploadDir)s/%(repository)s`"\n'
    'PARENTHASH=`echo $PARENTDIR | sed -e "s|.*-||"`\n'
    'if [ ! "X$PARENTHASH" = X%(originalParentHash)s ]; then\n'
    '  rm -f %(uploadDir)s/$TARGETREPO.next-$UNIQUEID\n' 
    '  rm -rf $UNIQUEDIR\n' 
    '  echo "Parent mismatch $PARENTHASH %(originalParentHash)s."\n'
    '  exit 1\n'
    'fi\n'
    'mv -T %(uploadDir)s/$TARGETREPO.next-$UNIQUEID %(uploadDir)s/$TARGETREPO\n'
    'rm -rf %(uploadDir)s/$TARGETREPO.next-$UNIQUEID\n'
    'rm -rf $UNIQUEDIR\n',
    serverAptEnv=opts.serverAptEnv,
    cmsplatf=opts.architecture,
    tmpUploadDir=tmpUploadDir,
    subsystems=subsystems,
    syncBack=opts.syncBack,
    originalParentHash=parentHash,
    **repoInfo)
  (err, msg) = executeScript(opts, createRepo, False)
  if err:
    log("Parallel upload session succeded before us. Starting uploading procedure from scratch. This will check again build area consistency.")
    if opts.debug:
      log(msg)
    return False
  log("Upload successfully finished")
  return True
            
def remove_help ():
    log ("cmsBuild delete <package>")

allOk = lambda x,y: x and y
someOk = lambda x,y: x or y

def doDeprecateLocal (opts, args, factory):
  fullPackageList = allPackagesToUpload(args, factory)
  executeScript(opts, deprecateLocalCommand(opts, fullPackageList))

def getSpecRepository (url, options):
    destTmpDir = abspath (options.tempDirPrefix)
    destTgz = join(destTmpDir,"CMSDIST.tgz")
    destDir = abspath ("CMSDIST")
    if exists (destTgz):
        unlink (destTgz)
    if "module=" not in url:
        url = url + "&module="+ DEFAULT_SPEC_REPOSITORY_MODULE
    if "strategy=" not in url:
        url = url + "&strategy=checkout"        
    if "timestamp" not in url:
        url = url + "&timestamp=%s" % strftime ("%Y%m%d%H%M%S") 
    if "output=" not in url:
        url = url + "&output=/%s.tgz" % DEFAULT_SPEC_REPOSITORY_MODULE        
    download (url, destTmpDir, options)
    command = "tar xzvf %(destTgz)s CMSDIST " % locals ()
    message = "ERROR: Error while trying to download CMSDIST from a remote source:"
    ok = executeWithErrorCheck (command, message)
    if not ok:
        sys.exit (1)
    return destDir

class CaseSensitiveConfigParser (ConfigParser):
    def optionxform (self, option):
        return option

g_originalEnvironment = {}

def saveOriginalEnvironment ():
    """ Saves the environment.
    """
    for key, value in environ.iteritems ():
        if key in ["SHELL"]:
            continue
        g_originalEnvironment[key] = value

def restoreOriginalEnvironment ():
    """ Restore the environment.
    """
    assert (g_originalEnvironment)
    for key, value in g_originalEnvironment.iteritems ():
        if key in ["SHELL"]:
            continue
        environ[key] = value
    for key in environ.iterkeys ():
        if key in ["SHELL"]:
            continue
        if key not in g_originalEnvironment.iterkeys ():
            environ[key] = ""

def rpmCacheUpdate(opts):
    if hasattr (opts, "bootstrap") and opts.bootstrap == True:
        rpm_db_cache.clear()
        rpmQueryCommand = "rpm -qa --queryformat '%{NAME} %{RELEASE}\n'"
        error, output = getstatusoutput (rpmQueryCommand)
        if error:
            die("Error while executing rpm -qa.\n%s" % output)
        for p in output.split("\n"):
            n,r = p.split(" ",1)
            rpm_db_cache[n]=r
    return

def setupAptEnvironment(environmentInitSh):
  if not exists(environmentInitSh):
    fatal("Error while bootstrapping. File not found %s." % environmentInitSh)
  for parts in pickle.load(popen('. %s; unset module; python -c "import os, sys, pickle; pickle.dump(os.environ, sys.stdout)"' % environmentInitSh)).iteritems():
    if len(parts) != 2:
      fatal("Unable to interpret environment. Malformed line %s" % line)
    environ.__setitem__(*parts)
  error, aptLocation = getstatusoutput("which apt-get")
  if error or not aptLocation.startswith(opts.workDir):
    fatal("Cannot find a working apt-get in bootstrapped area %s" % opts.workDir)

if __name__ == "__main__":
    # Stop processing if SCRAM runtime env is set to avoid picking up
    # python from cms external area
    if environ.has_key("SCRAMRT_SET"):
        fatal ("You have SCRAM runtime environment set. Please run it from a fresh shell.")
    # Avoid locale related issues with gcc output.
    environ["LANG"] = "C"
    opts, args = parseOptions()

    packages = {}
    #tags_cache = TagCacheAptImpl (opts)
    try:
      # We only support three different workflows:
      #
      # * cmsBuild [--no-bootstrap] build <package> [<additional packages>]
      # * cmsBuild [--no-bootstrap] [--sync-back] upload <package> [<additional packages>]
      # * cmsBuild [--no-bootstrap] deprecate-local <package> [<additional packages>]
      #
      # where all workflows include a bootstrap step (unless --no-bootstrap) is
      # specified and upload implies building the same packages.
      commandSpec = []
      
      if opts.bootstrap:
        commandSpec.append("bootstrap")

      if not exists(abspath(opts.cmsdist)) and not "://" in opts.cmsdist:
        cmsdist = abspath(opts.cmsdist)
        fatal("Wrong path specified for CMSDIST: %s.\nNothing done." % cmsdist)

      if args[0] != "deprecate-local":
        commandSpec.append("build")

      if args[0] != "build":
        commandSpec.append(args[0])

      log("The following commands will be executed:\n %s" % 
          "\n".join (["* %s" % spec for spec in commandSpec]), DEBUG)
        
      # Create the work area and a tmp dir for it it does not exists and cd to it.
      opts.tempdir = join(opts.workDir, opts.tempDirPrefix)
      if not exists(opts.tempdir):
        makedirs(opts.tempdir)
      chdir (opts.workDir)
      
      #Process level lock: To make sure that there is only one cmsBuild process is running for this work area
      procLock = cmsLock(opts.workDir)
      if not procLock:
          fatal ("There is already a cmsBuild process running for workdir %s." % opts.workDir)

      executeWithErrorCheck ("rm -rf %s" % join(opts.workDir, opts.tempDirPrefix, "BUILDROOT"),"Unable to remove BUILDROOT directory.")
	  
      # Checkout if a tag is specified for CMSDIST. If it is checkout the package
      # in a temporary directory and keep track of the path to it.
      if "://" in opts.cmsdist:
        opts.cmsdist = getSpecRepository(opts.cmsdist, opts)
      # Syntactic sugar to be able to say:
      #
      # cmsBuild build CMSSW_X_Y_Z
      #
      # which modifies the cmssw.spec in-place and builds it.
      cmsswCommands = [x for x in args if x.startswith("CMSSW_")] 
      if len(cmsswCommands) > 1:
        fatal("You can only build one version of CMSSW at the time")
      elif len(cmsswCommands) == 1:
        cmsswVersion = cmsswCommands[0]
        args.remove(cmsswVersion)
        args.append("cmssw")
        cmsswSpec = join(opts.cmsdist, "cmssw.spec")
        cmd = "perl -p -i -e 's/### RPM cms cmssw.*/### RPM cms cmssw %s/' %s"
        error, output = getstatusoutput(cmd % (cmsswVersion, cmsswSpec))
        if error:
          fatal("Unable to modify cmssw.spec as requested.")
      
      if not exists(opts.cmsdist):
        fatal("A CMSDIST checkout (or a tag) is required in order to build.")
      # Make sure that the build architecture and the actual platform we
      # are running on are compatible.
      if opts.architecture.startswith("osx"):
        if not sys.platform == "darwin":
          fatal("Cannot build architecture %s on linux. Did you mean %s?" % (opts.architecture, opts.architecture.replace("osx106", "slc5")))
      if opts.architecture.startswith("slc"):
        if not sys.platform.startswith("linux"):
          fatal("Cannot build architecture %s on osx. Did you mean %s?" % (opts.architecture, opts.architecture.replace("slc5", "osx106")))
      
      # Force the architecture and extract ancillary options like the compiler 
      # version, the compiler name, whether to force 32 bit builds on linux and
      # whether the system compiler should being used.
      # Cross check that:
      # * If on macosx or in online, check that the system compiler has the same
      #   version as the one specified in the architecture string.
      # * If we are not using the system compiler, a spec file with the 
      #   compiler name exists and matches the architecture.
      m = re.match("[^_]+_[^_]+_([a-z]+)([0-9]+)", opts.architecture)
      if not m:
        fatal("Malformed architecture string.")
      opts.compilerName, compactCompilerVersion = m.groups()
      opts.compilerVersion = ".".join([x for x in compactCompilerVersion])
      
      # Check validity of the system compiler.
      if opts.compilerName == "gcc" and sys.platform == "darwin":
        if detectCompilerVersion("gcc") == opts.compilerVersion:
          opts.systemCompiler = True
        else:
          opts.systemCompiler = False
      else:
        opts.systemCompiler = False
      
      # Check validity of the spec file.
      if not opts.systemCompiler:
        pathname = join (opts.cmsdist, "%s.spec" % opts.compilerName)
        try:
          lines = open(pathname).readlines()
        except IOError, e:
          fatal("The spec %s does not match the requested architecture %s." % (pathname, opts.architecture))
        (group, name, version) = parseRPMLine(lines, opts)
        version = version.split ("-")[0]
        if name != opts.compilerName or version != opts.compilerVersion:
          msg = "The compiler %s-%s (specified in %s) differs from %s-%s (specified by the architecture %s)."
          fatal(msg % (name, version, pathname, opts.compilerName,
                       opts.compilerVersion, opts.architecture))
      
      arch_data = opts.architecture.split("_",2)
      arch_data[2] = re.sub('^[a-z]+','',arch_data[2])
      opts.rpmQueryDefines = '--define "cmscompilerv  %s" --define "cmsos %s"' % (arch_data[2], "_".join(arch_data[0:2]))
      environ["SCRAM_ARCH"] = opts.architecture
      environ["VO_CMS_SW_DIR"] = opts.workDir
      
      # Tags cache always needs to be there.
      tags_cache = TagCacheAptImpl(opts)
      successfulTransaction = False 
      successfulBootstrapEnv = False
      while not successfulTransaction:
        successfulTransaction = True 
        checksums_cache = {}
        factory = PackageFactory(opts)
        for command in commandSpec:
          if command == "bootstrap":
            # If the apt-init.sh file is missing, do the bootstrap,
            # parse the output to find the init.sh and link it to
            # apt-init.sh, so that next time this stage will we skipped.
            #
            # Source the apt-init.sh file and setup the environment,
            # update the apt cache.
            aptInitSh = join(opts.workDir, opts.architecture, "apt-init.sh") 
            if not exists(aptInitSh):
              environmentInitSh = bootstrap(opts)
              symlink(environmentInitSh.replace(opts.workDir, ".."), aptInitSh)
            if not successfulBootstrapEnv:
              setupAptEnvironment(aptInitSh)
              successfulBootstrapEnv = True
            tags_cache.update()
          elif command == "deprecate-local":
            doDeprecateLocal(opts, args[1:], factory)
          elif command == "build":
            build(opts, args[1:], factory)
          elif command == "upload":
            # We continue trying unless some fatal error occurs (which exits
            # immediately).  or the remote repository is not consistent with the
            # current state.
            successfulTransaction = upload(opts, args[1:], factory)
            if not successfulTransaction:
              log("Looks like repository changed while we were building. Trying again.")
    except KeyboardInterrupt:
        log ("User requested to abort. Exiting.")
        sys.exit (1)
    except RpmBuildFailed, e:
        log ("Package %s could not build. Final lines of the build log are: " % e.pkg.name)
        logfile = open("%s/log" % e.pkg.builddir).readlines ()
        for line in logfile[-10:]:
          log(line)
        sys.exit (1)
    except RpmInstallFailed, e:
        log("The installation of %s could not be completed for the following reason:")
        log(e.why)
        sys.exit (1)
    except NotCorrectlyBootstrapped, e:
        log("""FATAL: While it seems that you have run bootstrap.sh, 
               the bootstrapped area actually looks broken for the following reason:""")
        log (e.why)
        sys.exit (1)
    except FileNotFound, e:
        log ("""ERROR! File not found: %(filename)s""" % e.__dict__)
        sys.exit (1)
    except PlatformDetectionError, e:
        log("""ERROR: Unable to detect the architecture:""")
        log("%s" % e)
        sys.exit (1)