test build with meta-swupd and new images

README.rst

@@ -21,6 +21,7 @@ Ostro OS:
 - meta-java
 - meta-soletta
 - meta-openembedded
+- meta-swupd
 
 The top-level directory comes from openembedded-core and meta-ostro
 (including this README.rst), everything else is in its own

bitbake/LICENSE

@@ -8,3 +8,5 @@ Foundation and individual contributors.
 * Twitter Bootstrap (including Glyphicons), redistributed under the Apache License 2.0.
 
 * jQuery is redistributed under the MIT license.
+
+* QUnit is redistributed under the MIT license.

bitbake/bin/bitbake

@@ -35,7 +35,7 @@ except RuntimeError as exc:
 from bb import cookerdata
 from bb.main import bitbake_main, BitBakeConfigParameters, BBMainException
 
-__version__ = "1.29.1"
+__version__ = "1.30.0"
 
 if __name__ == "__main__":
     if __version__ != bb.__version__:

bitbake/bin/toaster

@@ -17,7 +17,7 @@
 # You should have received a copy of the GNU General Public License
 # along with this program. If not, see http://www.gnu.org/licenses/.
 
-# Usage: source toaster [start|stop|restart-bitbake]
+# Usage: source toaster [start|stop]
 #                       [webport=<port>] [noui] [noweb]
 
 # Helper function to kill a background toaster development server
@@ -95,42 +95,18 @@ stop_system()
         kill `cat ${BUILDDIR}/.toasterui.pid` 2>/dev/null
         rm ${BUILDDIR}/.toasterui.pid
     fi
-    stop_bitbake
     webserverKillAll
     # unset exported variables
     unset DATABASE_URL
     unset TOASTER_CONF
     unset TOASTER_DIR
+    unset BITBAKE_UI
+    unset BBBASEDIR
     trap - SIGHUP
     #trap - SIGCHLD
     INSTOPSYSTEM=0
 }
 
-start_bitbake() {
-    unset BBSERVER
-    bitbake --read conf/toaster.conf --server-only -t xmlrpc -B 0.0.0.0:0
-    if [ $? -ne 0 ]; then
-        echo "Bitbake server start failed"
-        return 1
-    fi
-    export BBSERVER=0.0.0.0:-1
-    export DATABASE_URL=`$MANAGE get-dburl`
-    # we start the TOASTERUI only if not inhibited
-    if [ $NOTOASTERUI -eq 0 ]; then
-        bitbake --observe-only -u toasterui --remote-server=$BBSERVER -t xmlrpc \
-            >>${BUILDDIR}/toaster_ui.log 2>&1 \
-            & echo $! >${BUILDDIR}/.toasterui.pid
-    fi
-    return 0
-}
-
-stop_bitbake() {
-    BBSERVER=0.0.0.0:-1 bitbake -m
-    unset BBSERVER
-    # force stop any misbehaving bitbake server
-    lsof -t bitbake.lock | awk '{print $2}' | grep "[0-9]\+" | xargs -n1 -r kill
-}
-
 verify_prereq() {
     # Verify Django version
     reqfile=$(python -c "import os; print os.path.realpath('$BBBASEDIR/toaster-requirements.txt')")
@@ -156,7 +132,7 @@ else
     TOASTER=$0
 fi
 
-BBBASEDIR=`dirname $TOASTER`/..
+export BBBASEDIR=`dirname $TOASTER`/..
 MANAGE=$BBBASEDIR/lib/toaster/manage.py
 OEROOT=`dirname $TOASTER`/../..
 
@@ -201,15 +177,11 @@ fi
 # make sure that the toaster.sqlite file doesn't default to `pwd` like it currently does.
 export TOASTER_DIR=`pwd`
 
-NOTOASTERUI=0
 WEBSERVER=1
 WEB_PORT="8000"
 unset CMD
 for param in $*; do
     case $param in
-    noui )
-            NOTOASTERUI=1
-    ;;
     noweb )
             WEBSERVER=0
     ;;
@@ -229,15 +201,6 @@ if [ `basename \"$0\"` = `basename \"${TOASTER}\"` ]; then
     return 1
 fi
 
-if [ "$1" = 'restart-bitbake' ] ; then
-    stop_bitbake
-    sleep 1
-    start_bitbake
-    rc=$?
-    sleep 3
-    return $rc
-fi
-
 verify_prereq || return 1
 
 # We make sure we're running in the current shell and in a good environment
@@ -281,16 +244,6 @@ case $CMD in
              $MANAGE checksocket "0.0.0.0:$WEB_PORT" || return 1
         fi
 
-        # Make sure it's safe to start by checking bitbake lock
-        if [ -e $BUILDDIR/bitbake.lock ]; then
-            python -c "import fcntl; fcntl.flock(open(\"$BUILDDIR/bitbake.lock\"), fcntl.LOCK_EX|fcntl.LOCK_NB)" 2>/dev/null
-            if [ $? -ne 0 ] ; then
-                echo "Error: bitbake lock state error. File locks show that the system is on." 1>&2
-                echo "Please wait for the current build to finish, stop and then start the system again." 1>&2
-                return 3
-            fi
-        fi
-
         # kill Toaster web server if it's alive
         if [ -e $BUILDDIR/.toastermain.pid ] && kill -0 `cat $BUILDDIR/.toastermain.pid`; then
             echo "Warning: bitbake appears to be dead, but the Toaster web server is running." 1>&2
@@ -300,30 +253,21 @@ case $CMD in
         fi
 
         # Create configuration file
-        conf=${BUILDDIR}/conf/toaster.conf
-        echo "# Created by toaster start script" > $conf
-        echo "INHERIT+=\"toaster buildhistory\"" >> $conf
+        conf=${BUILDDIR}/conf/local.conf
+        line='INHERIT+="toaster buildhistory"'
+        grep -q "$line" $conf || echo $line >> $conf
 
         if [ $WEBSERVER -gt 0 ] && ! webserverStartAll; then
             echo "Failed ${CMD}."
             return 4
         fi
-        start_bitbake
-        if [ $? -eq 0 ]; then
-            $MANAGE runbuilds & echo $! >${BUILDDIR}/.runbuilds.pid
-            # set fail safe stop system on terminal exit
-            trap stop_system SIGHUP
-            echo "Successful ${CMD}."
-            return 0
-        else
-            # failed start, do stop
-            stop_system
-            echo "Failed ${CMD}."
-            return 1
-        fi
-        # stop system on terminal exit
-        set -o monitor
+        export BITBAKE_UI='toasterui'
+        export DATABASE_URL=`$MANAGE get-dburl`
+        $MANAGE runbuilds & echo $! >${BUILDDIR}/.runbuilds.pid
+        # set fail safe stop system on terminal exit
         trap stop_system SIGHUP
+        echo "Successful ${CMD}."
+        return 0
     ;;
     stop )
         stop_system

bitbake/lib/bb/__init__.py

@@ -21,7 +21,7 @@
 # with this program; if not, write to the Free Software Foundation, Inc.,
 # 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
 
-__version__ = "1.29.1"
+__version__ = "1.30.0"
 
 import sys
 if sys.version_info < (2, 7, 3):

bitbake/lib/bb/fetch2/git.py

@@ -262,23 +262,7 @@ def unpack(self, ud, destdir, d):
         if ud.bareclone:
             cloneflags += " --mirror"
 
-        # Versions of git prior to 1.7.9.2 have issues where foo.git and foo get confused
-        # and you end up with some horrible union of the two when you attempt to clone it
-        # The least invasive workaround seems to be a symlink to the real directory to
-        # fool git into ignoring any .git version that may also be present.
-        #
-        # The issue is fixed in more recent versions of git so we can drop this hack in future
-        # when that version becomes common enough.
-        clonedir = ud.clonedir
-        if not ud.path.endswith(".git"):
-            indirectiondir = destdir[:-1] + ".indirectionsymlink"
-            if os.path.exists(indirectiondir):
-                os.remove(indirectiondir)
-            bb.utils.mkdirhier(os.path.dirname(indirectiondir))
-            os.symlink(ud.clonedir, indirectiondir)
-            clonedir = indirectiondir
-
-        runfetchcmd("%s clone %s %s/ %s" % (ud.basecmd, cloneflags, clonedir, destdir), d)
+        runfetchcmd("%s clone %s %s/ %s" % (ud.basecmd, cloneflags, ud.clonedir, destdir), d)
         os.chdir(destdir)
         repourl = self._get_repo_url(ud)
         runfetchcmd("%s remote set-url origin %s" % (ud.basecmd, repourl), d)

bitbake/lib/bb/persist_data.py

@@ -201,6 +201,7 @@ def delValue(self, domain, key):
 def connect(database):
     connection = sqlite3.connect(database, timeout=5, isolation_level=None)
     connection.execute("pragma synchronous = off;")
+    connection.text_factory = str
     return connection
 
 def persist(domain, d):

bitbake/lib/bb/ui/buildinfohelper.py

@@ -218,6 +218,15 @@ def update_build_object(self, build, errors, warnings, taskfailures):
         assert isinstance(errors, int)
         assert isinstance(warnings, int)
 
+        if build.outcome == Build.CANCELLED:
+            return
+        try:
+            if build.buildrequest.state == BuildRequest.REQ_CANCELLING:
+                return
+        except AttributeError:
+            # We may not have a buildrequest if this is a command line build
+            pass
+
         outcome = Build.SUCCEEDED
         if errors or taskfailures:
             outcome = Build.FAILED
@@ -241,14 +250,16 @@ def update_task_object(self, build, task_name, recipe_name, task_stats):
             recipe__name = recipe_name
         )
 
-        task_to_update.started = self._timestamp_to_datetime(task_stats['started'])
-        task_to_update.ended = self._timestamp_to_datetime(task_stats['ended'])
-        task_to_update.elapsed_time = (task_stats['ended'] - task_stats['started'])
-        task_to_update.cpu_time_user = task_stats['cpu_time_user']
-        task_to_update.cpu_time_system = task_stats['cpu_time_system']
-        task_to_update.disk_io_read = task_stats['disk_io_read']
-        task_to_update.disk_io_write = task_stats['disk_io_write']
-        task_to_update.disk_io = task_stats['disk_io_read'] + task_stats['disk_io_write']
+        if 'started' in task_stats and 'ended' in task_stats:
+            task_to_update.started = self._timestamp_to_datetime(task_stats['started'])
+            task_to_update.ended = self._timestamp_to_datetime(task_stats['ended'])
+            task_to_update.elapsed_time = (task_stats['ended'] - task_stats['started'])
+        task_to_update.cpu_time_user = task_stats.get('cpu_time_user')
+        task_to_update.cpu_time_system = task_stats.get('cpu_time_system')
+        if 'disk_io_read' in task_stats and 'disk_io_write' in task_stats:
+            task_to_update.disk_io_read = task_stats['disk_io_read']
+            task_to_update.disk_io_write = task_stats['disk_io_write']
+            task_to_update.disk_io = task_stats['disk_io_read'] + task_stats['disk_io_write']
 
         task_to_update.save()
 
@@ -483,7 +494,7 @@ def save_target_file_information(self, build_obj, target_obj, filedata):
             parent_obj = self._cached_get(Target_File, target = target_obj, path = parent_path, inodetype = Target_File.ITYPE_DIRECTORY)
             tf_obj = Target_File.objects.create(
                         target = target_obj,
-                        path = path,
+                        path = unicode(path, 'utf-8'),
                         size = size,
                         inodetype = Target_File.ITYPE_DIRECTORY,
                         permission = permission,
@@ -508,7 +519,7 @@ def save_target_file_information(self, build_obj, target_obj, filedata):
 
             tf_obj = Target_File.objects.create(
                         target = target_obj,
-                        path = path,
+                        path = unicode(path, 'utf-8'),
                         size = size,
                         inodetype = inodetype,
                         permission = permission,
@@ -539,7 +550,9 @@ def save_target_file_information(self, build_obj, target_obj, filedata):
                 filetarget_path = "/".join(fcpl)
 
             try:
-                filetarget_obj = Target_File.objects.get(target = target_obj, path = filetarget_path)
+                filetarget_obj = Target_File.objects.get(
+                                     target = target_obj,
+                                     path = unicode(filetarget_path, 'utf-8'))
             except Target_File.DoesNotExist:
                 # we might have an invalid link; no way to detect this. just set it to None
                 filetarget_obj = None
@@ -548,7 +561,7 @@ def save_target_file_information(self, build_obj, target_obj, filedata):
 
             tf_obj = Target_File.objects.create(
                         target = target_obj,
-                        path = path,
+                        path = unicode(path, 'utf-8'),
                         size = size,
                         inodetype = Target_File.ITYPE_SYMLINK,
                         permission = permission,
@@ -849,7 +862,7 @@ class BuildInfoHelper(object):
     # pylint: disable=bad-continuation
     # we do not follow the python conventions for continuation indentation due to long lines here
 
-    def __init__(self, server, has_build_history = False):
+    def __init__(self, server, has_build_history = False, brbe = None):
         self.internal_state = {}
         self.internal_state['taskdata'] = {}
         self.internal_state['targets'] = []
@@ -865,7 +878,7 @@ def __init__(self, server, has_build_history = False):
 
         # this is set for Toaster-triggered builds by localhostbecontroller
         # via toasterui
-        self.brbe = None
+        self.brbe = brbe
 
         self.project = None
 
@@ -1220,8 +1233,8 @@ def store_target_package_data(self, event):
         for target in self.internal_state['targets']:
             if target.is_image:
                 pkgdata = BuildInfoHelper._get_data_from_event(event)['pkgdata']
-                imgdata = BuildInfoHelper._get_data_from_event(event)['imgdata'][target.target]
-                filedata = BuildInfoHelper._get_data_from_event(event)['filedata'][target.target]
+                imgdata = BuildInfoHelper._get_data_from_event(event)['imgdata'].get(target.target, {})
+                filedata = BuildInfoHelper._get_data_from_event(event)['filedata'].get(target.target, {})
 
                 try:
                     self.orm_wrapper.save_target_package_information(self.internal_state['build'], target, imgdata, pkgdata, self.internal_state['recipes'], built_package=True)
@@ -1244,7 +1257,6 @@ def store_dependency_information(self, event):
         assert 'layer-priorities' in event._depgraph
         assert 'pn' in event._depgraph
         assert 'tdepends' in event._depgraph
-        assert 'providermap' in event._depgraph
 
         errormsg = ""
 
@@ -1330,7 +1342,7 @@ def store_dependency_information(self, event):
                 if dep in assume_provided:
                     continue
                 via = None
-                if dep in event._depgraph['providermap']:
+                if 'providermap' in event._depgraph and dep in event._depgraph['providermap']:
                     deprecipe = event._depgraph['providermap'][dep][0]
                     dependency = self.internal_state['recipes'][deprecipe]
                     via = Provides.objects.get_or_create(name=dep,
@@ -1404,9 +1416,18 @@ def _store_build_done(self, errorcode):
         be.lock = BuildEnvironment.LOCK_LOCK
         be.save()
         br = BuildRequest.objects.get(pk = br_id)
+
+        # if we're 'done' because we got cancelled update the build outcome
+        if br.state == BuildRequest.REQ_CANCELLING:
+            logger.info("Build cancelled")
+            br.build.outcome = Build.CANCELLED
+            br.build.save()
+            self.internal_state['build'] = br.build
+            errorcode = 0
+
         if errorcode == 0:
             # request archival of the project artifacts
-            br.state = BuildRequest.REQ_ARCHIVE
+            br.state = BuildRequest.REQ_COMPLETED
         else:
             br.state = BuildRequest.REQ_FAILED
         br.save()