Merge pull request #95 from pohly/meta-swupd

meta-swupd

meta-ostro/classes/image-dsk.bbclass

@@ -220,12 +220,11 @@ deltask do_rootfs_wicenv
 # All variables explicitly passed to image-iot.py.
 IMAGE_DSK_VARIABLES = " \
     APPEND \
-    BPN \
     DEPLOY_DIR_IMAGE \
     DSK_IMAGE_LAYOUT \
+    IMAGE_LINK_NAME \
     IMAGE_NAME \
     IMAGE_ROOTFS \
-    MACHINE \
     ROOTFS_TYPE \
     ROOTFS_PARTUUID_VALUE \
     PARTITION_TYPE_EFI \

meta-ostro/classes/ostro-sanity.bbclass

@@ -8,3 +8,57 @@ python ostro_sanity_check_eventhandler() {
 production and development images. See the comments in local.conf.sample
 and doc/howtos/building-images.rst.''')
 }
+
+# /run, /proc, /var/volatile and /dev only get mounted at runtime.
+OSTRO_QA_IMAGE_SYMLINK_WHITELIST = " \
+    /dev/null \
+    /proc/mounts \
+    /run/lock \
+    /run/resolv.conf \
+    /var/volatile/log \
+    /var/volatile/tmp \
+"
+
+# Additional image checks.
+python ostro_qa_image () {
+    qa_sane = True
+
+    rootfs = d.getVar("IMAGE_ROOTFS", True)
+
+    def resolve_links(target, root):
+        if not target.startswith('/'):
+            target = os.path.normpath(os.path.join(root, target))
+        else:
+            # Absolute links are in fact relative to the rootfs.
+            # Can't use os.path.join() here, it skips the
+            # components before absolute paths.
+            target = os.path.normpath(rootfs + target)
+        if os.path.islink(target):
+            root = os.path.dirname(target)
+            target = os.readlink(target)
+            target = resolve_links(target, root)
+        return target
+
+    # Check for dangling symlinks. One common reason for them
+    # in swupd images is update-alternatives where the alternative
+    # that gets chosen in the mega image then is not installed
+    # in a sub-image.
+    #
+    # Some allowed cases are whitelisted.
+    whitelist = d.getVar('OSTRO_QA_IMAGE_SYMLINK_WHITELIST', True).split()
+    for root, dirs, files in os.walk(rootfs):
+        for entry in files + dirs:
+            path = os.path.join(root, entry)
+            if os.path.islink(path):
+                target = os.readlink(path)
+                final_target = resolve_links(target, root)
+                if not os.path.exists(final_target) and not final_target[len(rootfs):] in whitelist:
+                    bb.error("Dangling symlink: %s -> %s -> %s does not resolve to a valid filesystem entry." %
+                             (path, target, final_target))
+                    qa_sane = False
+
+    if not qa_sane:
+        bb.fatal("Fatal QA errors found, failing task.")
+}
+
+do_image[postfuncs] += "ostro_qa_image"

meta-ostro/classes/ostro-update-alternatives.bbclass

@@ -0,0 +1,32 @@
+# This code only runs in recipes which inherit update-alternatives.bbclass.
+# It manipulates the postinst scripts created by update-alternatives.bbclass
+# so that "update-alternatives --install" calls take
+# ${ALTERNATIVE_PRIORITY_<package name in upper case and with underscores>:-<def priority>}
+# as priority parameter, i.e. it becomes possible to override the default priority
+# set in the recipe at install time via env variables.
+def ostro_manipulate_postinst_alternatives(d):
+    import re
+
+    pn = d.getVar('BPN', True)
+    suffix = (''.join([x if x.isalnum() else '_' for x in pn])).upper()
+    replacements = set()
+    def new_alt_priority(m):
+        replacement = '${ALTERNATIVE_PRIORITY_%s:-%s}' % (
+                         suffix,
+                         m.group(2),
+                       )
+        if not replacement in replacements:
+            bb.note('adding support for %s update-alternatives image variable' % replacement)
+            replacements.add(replacement)
+        return m.group(1) + replacement
+
+    regex = re.compile(r'''(update-alternatives\s+--install\s+\S+\s+\S+\s+\S+\s+)(\S+)''')
+    for pkg in (d.getVar('PACKAGES', True) or "").split():
+        postinst = d.getVar('pkg_postinst_%s' % pkg, True)
+        if postinst:
+            postinst = regex.sub(new_alt_priority, postinst)
+            d.setVar('pkg_postinst_%s' % pkg, postinst)
+
+python populate_packages_updatealternatives_append () {
+    ostro_manipulate_postinst_alternatives(d)
+}

meta-ostro/conf/bblayers.conf.sample

@@ -1,6 +1,6 @@
 # LAYER_CONF_VERSION is increased each time build/conf/bblayers.conf
 # changes incompatibly
-LCONF_VERSION = "7"
+LCONF_VERSION = "8"
 
 BBPATH = "${TOPDIR}"
 BBFILES ?= ""
@@ -23,6 +23,7 @@ OSTRO_LAYERS = " \
   ##OEROOT##/meta-iotqa \
   ##OEROOT##/meta-appfw \
   ##OEROOT##/meta-security-isafw \
+  ##OEROOT##/meta-swupd \
   "
 
 # The following layers are available in the combined repository.

meta-ostro/conf/conf-notes.txt

@@ -1,10 +1,7 @@
 Common targets are:
-    ostro-image
-    ostro-image-dev
-    core-image-minimal
-    meta-toolchain
-    adt-installer
-    meta-ide-support
+    ostro-image-noswupd (when building without swupd,
+                         the recommended mode for local image building)
+    ostro-image-swupd-reference (when building with swupd)
 
 We recommend to use VirtualBox if you wish to run an Ostro image in a
 virtual environment. Please check the doc/howtos/booting-and-installation.rst

meta-ostro/conf/distro/include/ostro-os-development.inc

@@ -1,8 +1,10 @@
 # Use the pre-generated keys for IMA signing.
 IMA_EVM_KEY_DIR = "${IMA_EVM_BASE}/data/debug-keys"
 
-# Enable local root access.
-OSTRO_EXTRA_IMAGE_FEATURES += "autologin"
+# Enable local root access. Always use _append, to
+# avoid getting this change overwritten by a
+# OSTRO_IMAGE_EXTRA_FEATURES = "foo" in local.conf.
+OSTRO_IMAGE_EXTRA_FEATURES_append = " autologin"
 
 # Mark images as "development" by extending /etc/motd.
 OSTRO_EXTRA_MOTD () {
@@ -12,6 +14,14 @@ OSTRO_EXTRA_MOTD () {
 ************************************
 }
 
+# Extend the default package selection and image configuration of the
+# no-swupd "ostro-image" to match the content of the swupd
+# "ostro-image-swupd-reference".
+OSTRO_IMAGE_NOSWUPD_EXTRA_FEATURES_append = " ${OSTRO_DEVELOPMENT_EXTRA_FEATURES}"
+OSTRO_IMAGE_NOSWUPD_EXTRA_INSTALL_append = " ${OSTRO_DEVELOPMENT_EXTRA_INSTALL}"
+OSTRO_DEVELOPMENT_EXTRA_FEATURES ?= "${OSTRO_IMAGE_FEATURES_REFERENCE}"
+OSTRO_DEVELOPMENT_EXTRA_INSTALL ?= "${OSTRO_IMAGE_INSTALL_REFERENCE}"
+
 # Everything ready, avoid initial sanity check.
 OSTRO_IMAGE_BUILD_MODE_SELECTED = "1"
 

meta-ostro/conf/distro/include/ostroproject-ci.inc

@@ -31,7 +31,7 @@ SDKMACHINE ?= "x86_64"
 # Generate source mirror tarballs:
 BB_GENERATE_MIRROR_TARBALLS = "1"
 
-# Include the devel settings
+# Configure images in "development mode".
 require conf/distro/include/ostro-os-development.inc
 
 # XXX Drop the VM hack after taking care also of the non UEFI devices
@@ -45,18 +45,33 @@ OSTRO_VM_IMAGE_TYPES = "dsk.xz dsk.zip dsk.vdi.zip dsk.bmap dsk.xz.sha256sum dsk
 # Any other symbols would be skipped in parser.
 #
 # Following targets would be used to perform default build task:
-OSTROPROJECT_CI_BUILD_TARGETS="${DISTRO}-image ${DISTRO}-image-dev"
+OSTROPROJECT_CI_BUILD_TARGETS="ostro-image-noswupd ostro-image-swupd ostro-image-swupd-dev ostro-image-swupd-reference"
 # Following targets would be executed with do_populate_sdk task
 OSTROPROJECT_CI_SDK_TARGETS=""
-# Following targets would be executed with do_populate_sdk_ext task
-OSTROPROJECT_CI_ESDK_TARGETS="${DISTRO}-image"
+# Following targets would be executed with do_populate_sdk_ext task.
+# Uses the fully-populated image to ensure that the eSDK is as complete
+# as possible.
+OSTROPROJECT_CI_ESDK_TARGETS="ostro-image-swupd-all"
 # Following targets would be executed with do_test_iot_export task
-OSTROPROJECT_CI_TEST_EXPORT_TARGETS="${DISTRO}-image"
+OSTROPROJECT_CI_TEST_EXPORT_TARGETS="ostro-image-swupd-dev"
 # Execute automatic tests for following images with corresponding
 # test suite and test files.
 # Space separated list of tuples, each should in format:
 # <image_name>,<testsuite_name>,<testfiles_name>,$MACHINE
-OSTROPROJECT_CI_TEST_RUNS="${DISTRO}-image,iot-testsuite.tar.gz,iot-testfiles.${MACHINE}.tar.gz,${MACHINE}"
+OSTROPROJECT_CI_TEST_RUNS="ostro-image-swupd-dev,iot-testsuite.tar.gz,iot-testfiles.${MACHINE}.tar.gz,${MACHINE}"
+
+# In the CI system, the integer OS version used for swupd is derived
+# from the Jenkins build number part of the BUILD_ID. The conversion
+# to int and back to string acts as sanity check that we really get a
+# number out of the BUILD_ID.
+#
+# BUILD_ID is expected to have this format:
+# BUILD_ID = "<parent-build-timestamp (of Jenkins top job)>" + "-build-" + "<jenkins-top-job-build-number>"
+#
+# We multiply it by ten, to ensure that the version space has some gap
+# for minor updates. How to actually build minor updates with Jenkins
+# still needs to be determined.
+OS_VERSION ?= "${@ str(int('${BUILD_ID}'.split('-')[-1]) * 10) }"
 
 #
 # Dymamic section.

meta-ostro/conf/distro/ostro.conf

@@ -12,6 +12,52 @@ SDK_UPDATE_URL = "\
 		${MACHINE}\
 "
 
+# When building in a CI system with swupd enabled, OS_VERSION must be
+# set to a consistent value for all builds. See ostroproject-ci.inc
+# for an example how that works with Jenkins.
+#
+# In local builds, we only have ${DATETIME} as something that
+# increments automatically, but it is too large for an integer number
+# on 32 bit systems. Therefore we substract the 2016 as the initial
+# year in which Ostro OS started using swupd (assumes a 64 build host,
+# to avoid integer overflows in Python itself) and ignore the
+# seconds.
+#
+# The default behavior is to not rebuild just because OS_VERSION
+# changed. If that is desired, include in local.conf:
+#   VERSION_ID = "${OS_VERSION}"
+# This will cause the os-release package to be rebuilt each time
+# OS_VERSION changes, and that in turn causes an image and bundle
+# rebuild.
+#
+# For more predictable results is possible to set OS_RELEASE manually,
+# either in local.conf or in the environment like this:
+#   BB_ENV_EXTRAWHITE="$BB_ENV_EXTRAWHITE OS_VERSION" OS_VERSION=110 bitbake ...
+OS_VERSION ?= "${@ str(((int('${DATETIME}') - 20160000000000) / 100) * 10) }"
+OS_VERSION[vardepsexclude] += "DATETIME"
+
+# Ostro OS tries to build minimal images and thus prefers Busybox or
+# Toybox over coreutils or any other package providing similar
+# functionality. For technical reasons, this is crucial when building
+# swupd bundles: Busybox/Toybox must be picked consistently, even when
+# a bundle pulls in alternatives like coreutils which normally would
+# have a higher priority.
+#
+# This gets achieved in two ways:
+# 1. via ostro-update-alternatives.bbclass we modify all packages
+#    so that their update-alternatives priority can be modified
+#    during install time
+# 2. in ostro-image.bbclass we reconfigure those priorities
+#    as necessary
+#
+# This is still a change that is done consistently for all images,
+# whether they use swupd or not, to ensure that the result is
+# identical. Doing it at image creation time reduces the need
+# for recompilations of packages and opens up the possibility
+# for downstream developers to override the defaults while
+# reusing pre-compiled binaries from our sstate.
+INHERIT += "ostro-update-alternatives"
+
 MAINTAINER = "Ostro Dev <[email protected]>"
 
 TARGET_VENDOR = "-ostro"
@@ -58,12 +104,24 @@ DISTRO_FEATURES_DEFAULT_remove = "x11 3g"
 
 DISTRO_FEATURES ?= "${DISTRO_FEATURES_DEFAULT} ${DISTRO_FEATURES_LIBC} ${OSTRO_DEFAULT_DISTRO_FEATURES}"
 
+# Currently Ostro OS is still using swupd 2.x because that is what
+# has been tested with most. The newer 3.x will be tested right after
+# the initial swupd integration.
+PREFERRED_VERSION_swupd-client ?= "2.%"
+PREFERRED_VERSION_swupd-client-native ?= "2.%"
+PREFERRED_VERSION_swupd-server ?= "2.%"
+PREFERRED_VERSION_swupd-server-native ?= "2.%"
+
 # Use 4.4 kernel for meta-intel BSP MACHINEs
 # BeagleBone is still on 4.1 until meta-yocto-bsp switches it over
 PREFERRED_VERSION_linux-yocto_intel-corei7-64 ?= "4.4%"
 PREFERRED_VERSION_linux-yocto_intel-quark ?= "4.4%"
 PREFERRED_VERSION_linux-yocto_beaglebone ?= "4.1%"
 
+# Use UEFI-based "dsk" image format for machines supporting UEFI.
+# Defined here because it influences several different recipes.
+OSTRO_USE_DSK_IMAGES ?= "${@bb.utils.contains_any('MACHINE', 'intel-core2-32 intel-corei7-64 intel-quark', 'True', 'False', d)}"
+
 DISTRO_EXTRA_RDEPENDS += " ${OSTRO_DEFAULT_EXTRA_RDEPENDS}"
 DISTRO_EXTRA_RRECOMMENDS += " ${OSTRO_DEFAULT_EXTRA_RRECOMMENDS}"
 
@@ -82,8 +140,6 @@ DISTRO_FEATURES_BACKFILL_CONSIDERED = "sysvinit"
 require conf/distro/include/ostro_security_flags.inc
 require conf/distro/include/ostro-x11.inc
 
-OS_RELEASE_FIELDS_append = " BUILD_ID"
-
 # QA check settings - a little stricter than the OE-Core defaults
 WARN_TO_ERROR_QA = "already-stripped compile-host-path install-host-path \
                     installed-vs-shipped ldflags pn-overrides rpaths staticdev \
@@ -145,16 +201,24 @@ PNWHITELIST_meta-python += " \
 "
 PNWHITELIST_multimedia-layer += ""
 PNWHITELIST_networking-layer += " \
+    iftop \
     lowpan-tools \
+    traceroute \
 "
 PNWHITELIST_openembedded-layer += " \
-    gmock \
-    gtest \
-    libsocketcan \
+    atop \
     can-utils \
+    dash \
     giflib \
+    gmock \
+    gtest \
+    htop \
+    iotop \
     krb5 \
+    libmicrohttpd \
+    libsocketcan \
     pkcs11-helper \
+    vim \
 "
 PNWHITELIST_perl-layer += ""
 PNWHITELIST_ruby-layer += ""

meta-ostro/conf/layer.conf

@@ -20,7 +20,7 @@ BBFILE_PRIORITY_ostro = "6"
 LOCALCONF_VERSION = "2"
 
 # Same for LCONF_VERSION in bblayer.conf.sample.
-LAYER_CONF_VERSION = "7"
+LAYER_CONF_VERSION = "8"
 
 # The default error messages use shell meta* wildcards to find the
 # conf files which in ostro-os happen to match two files (one from

meta-ostro/conf/local.conf.sample

@@ -272,6 +272,12 @@ CONF_VERSION = "2"
 # are not secure.
 #
 # require conf/distro/include/ostro-os-development.inc
+#
+# Including that file reconfigures "ostro-image" to have the same
+# content and configuration as the published "ostro-image-swupd-reference".
+# Uncomment these lines if that is not desired:
+# OSTRO_DEVELOPMENT_EXTRA_FEATURES = ""
+# OSTRO_DEVELOPMENT_EXTRA_INSTALL = ""
 
 # When building production images, configure IMA signing
 # as described in meta-integrity/README.md if IMA is enabled

meta-ostro/lib/image-dsk.py

@@ -129,7 +129,7 @@ def do_dsk_image():
                      expand_vars('${IMAGE_NAME}-disk-layout.json'))
     disk_layout_file_link = \
         os.path.join(expand_vars("${DEPLOY_DIR_IMAGE}"),
-                     expand_vars('${BPN}-${MACHINE}-disk-layout.json'))
+                     expand_vars('${IMAGE_LINK_NAME}-disk-layout.json'))
     with open(disk_layout_file, 'w') as disk_layout:
         json.dump(obj=partition_table, fp=disk_layout,
                   indent=4, separators=(',', ': '))

meta-ostro/recipes-core/os-release/os-release.bbappend

@@ -0,0 +1,9 @@
+# In Ostro OS We only put mostly static values into the os-release
+# package. That avoids unnecessary recompilations.  Dynamic values
+# like DISTRO_VERSION (which in our case contain ${DATE}) and BUILD_ID
+# (includes ${DATETIME}) get patched to the current values in
+# ostro-image.bbclass.
+
+DISTRO_VERSION = "distro-version-to-be-added-during-image-creation"
+BUILD_ID = "build-id-to-be-added-during-image-creation"
+OS_RELEASE_FIELDS += "BUILD_ID"

meta-ostro/recipes-core/packagegroups/packagegroup-tools-interactive.bb

@@ -0,0 +1,30 @@
+SUMMARY = "Tools expected in an interactive shell"
+DESCRIPTION = "Tools listed here are meant to be useful when logged into a device \
+and working in the shell interactively or with some custom scripts. In production \
+images without shell access they are optional. Tools with a specific purpose like \
+development, profiling or program debugging are listed in separate package groups. \
+"
+LICENSE = "MIT"
+
+inherit packagegroup
+
+RDEPENDS_${PN} = " \
+    atop \
+    curl \
+    gawk \
+    htop \
+    iftop \
+    iotop \
+    iputils-arping \
+    iputils-clockdiff \
+    iputils-ping \
+    iputils-ping6 \
+    iputils-tracepath \
+    iputils-tracepath6 \
+    iputils-traceroute6 \
+    procps \
+    rsync \
+    traceroute \
+    vim \
+    wget \
+"