Add support for downloading artifacts from matrix builds

omego/artifacts.py

@@ -5,7 +5,7 @@
 import subprocess
 import logging
 
-from urllib2 import build_opener
+from urllib2 import build_opener, HTTPError
 import re
 
 from framework import Command, Stop
@@ -41,21 +41,16 @@ class Artifacts(object):
     def __init__(self, args):
 
         self.args = args
-        url = opener.open(args.build+"api/xml")
-        try:
-            log.debug('Fetching xml from %s code:%d', url.url, url.code)
-            if url.code != 200:
-                log.error('Failed to get CI XML from %s (code %d)',
-                          url.url, url.code)
-                raise Stop(20, 'Job lookup failed, is the job name correct?')
-            ci_xml = url.read()
-        finally:
-            url.close()
+        buildurl = args.build
 
-        root = XML(ci_xml)
+        root = self.read_xml(buildurl)
+        if root.tag == "matrixBuild":
+            runs = root.findall("./run/url")
+            buildurl = self.find_label_matches(r.text for r in runs)
+            root = self.read_xml(buildurl)
 
         artifacts = root.findall("./artifact")
-        base_url = args.build+"artifact/"
+        base_url = buildurl + "artifact/"
         if len(artifacts) <= 0:
             raise AttributeError(
                 "No artifacts, please check build on the CI server.")
@@ -70,6 +65,55 @@ def __init__(self, args):
                     setattr(self, key, rel_path)
                     pass
 
+    def read_xml(self, buildurl):
+        url = None
+        try:
+            url = opener.open(buildurl + 'api/xml')
+            log.debug('Fetching xml from %s code:%d', url.url, url.code)
+            if url.code != 200:
+                log.error('Failed to get CI XML from %s (code %d)',
+                          url.url, url.code)
+                raise Stop(20, 'Job lookup failed, is the job name correct?')
+            ci_xml = url.read()
+        except HTTPError as e:
+            log.error('Failed to get CI XML (%s)', e)
+            raise Stop(20, 'Job lookup failed, is the job name correct?')
+        finally:
+            if url:
+                url.close()
+
+        root = XML(ci_xml)
+        return root
+
+    def find_label_matches(self, urls):
+        required = set(self.args.labels.split(','))
+        if '' in required:
+            required.remove('')
+        log.debug('Searching for matrix runs matching: %s', required)
+        matches = []
+        for url in urls:
+            url_labels = self.label_list_parser(url)
+            if len(required.intersection(url_labels)) == len(required):
+                matches.append(url)
+
+        if len(matches) != 1:
+            log.error('Found %d matching matrix build runs: %s',
+                      len(matches), matches)
+            raise Stop(
+                30, 'Expected one matching run, found %d' % len(matches))
+        return matches[0]
+
+    def label_list_parser(self, url):
+        """
+        Extracts comma separate tag=value pairs from a string
+        Assumes all characters other than / and , are valid
+        """
+        labels = re.findall('([^/,]+=[^/,]+)', url)
+        slabels = set(labels)
+        if '' in slabels:
+            slabels.remove('')
+        return slabels
+
     @classmethod
     def get_artifacts_list(self):
         return {'server': r'OMERO\.server.*\.zip',
@@ -90,6 +134,9 @@ def download(self, component):
         filename = os.path.basename(componenturl)
         unzipped = filename.replace(".zip", "")
 
+        if self.args.dry_run:
+            return
+
         if os.path.exists(unzipped):
             return unzipped
 
@@ -153,8 +200,5 @@ def __call__(self, args):
                 log.debug("% 20s => %s" % (dest, replacement))
                 setattr(args, dest, replacement)
 
-        if args.dry_run:
-            return
-
         artifacts = Artifacts(args)
         artifacts.download(args.artifact)

omego/env.py

@@ -89,6 +89,8 @@ def __init__(self, parser):
         Add(group, "build",
             "http://%(ci)s/job/%(branch)s/lastSuccessfulBuild/",
             help="Full url of the Jenkins build containing the artifacts")
+        Add(group, "labels", "ICE=3.5",
+            help="Comma separated list of labels for matrix builds")
 
         # UNZIP TOOLS
         if WINDOWS:

omego/upgrade.py

@@ -112,7 +112,7 @@ def configure(self, noconfigure):
             from path import path
             old_grid = path(self.args.sym) / "etc" / "grid"
             old_cfg = old_grid / "config.xml"
-            if os.path.samefile(old_cfg, target):
+            if target.exists() and os.path.samefile(old_cfg, target):
                 # This likely is caused by the symlink being
                 # created early on an initial install.
                 pass

test/TestUpgrade.py

@@ -49,6 +49,10 @@ def testSkipunzip(self):
     def testUpgrade(self):
         self.upgrade("--unzipargs=-q", "--branch=OMERO-5.0-latest-ice34")
 
+    def testUpgradeMatrixBuild(self):
+        self.upgrade(
+            "--unzipargs=-q", "--branch=OMERO-5.1-latest", "--labels=ICE=3.4")
+
 if __name__ == '__main__':
     import logging
     logging.basicConfig()