Openstack cloudinit

bin/generate-buildenv

@@ -45,7 +45,7 @@ _git_state() {
 }
 
 _common() {
-    git_state $BT
+    _git_state $BT
     echo "turnkey_version $(cat /etc/turnkey_version)"
     echo "ami-id $(which ec2metadata && ec2metadata --ami-id || echo none)"
     echo "awscli $(python -c 'import awscli; print awscli.__version__')"

bin/openstack-bundle

@@ -33,6 +33,8 @@ rootfs=$1
 name=$(echo $rootfs | sed 's/.rootfs//')
 appname=$(echo $name |sed 's/turnkey-\(.*\)-[0-9].*/\1/')
 
+
+
 case "$appname" in
     canvas)           loopsize_padding=524288 ;;
     ejabberd)         loopsize_padding=524288 ;;
@@ -46,29 +48,82 @@ loopsize=$[$rootsize + $loopsize_padding]
 
 info "creating sparse loopback"
 dd if=/dev/null of=$rootfs.img bs=1 seek=${loopsize}K
-mkfs.ext4 -F -j $rootfs.img
 
-mkdir $rootfs.img.mount
-mount -o loop $rootfs.img $rootfs.img.mount
+info "creating partition"
+#inspired by package openstack-debian-images
+PARTED=/sbin/parted
+AMI_NAME=$rootfs.img
+${PARTED} -s ${AMI_NAME} mktable msdos
+${PARTED} -s -a optimal ${AMI_NAME} mkpart primary ext3 1Mi 100%
+${PARTED} -s ${AMI_NAME} set 1 boot on
+install-mbr ${AMI_NAME}
+RESULT_KPARTX=`kpartx -asv ${AMI_NAME} 2>&1`
+
+if echo "${RESULT_KPARTX}" | grep "^add map" ; then
+    LOOP_DEVICE=`echo ${RESULT_KPARTX} | cut -d" " -f3`
+    info "kpartx mounted using: ${LOOP_DEVICE}"
+else
+    fatal "It seems kpartx didn't mount the image correctly: exiting."
+fi
+
+cleanup(){
+    error=$?
+    [ ! -d "${MOUNT_DIR}" ] && return
+    echo
+    echo "error $error, umounting $MOUNT_DIR"
+    chroot ${MOUNT_DIR} umount /proc || true
+    chroot ${MOUNT_DIR} umount /sys || true
+    umount ${MOUNT_DIR}
+    rmdir ${MOUNT_DIR}
+    kpartx -d ${AMI_NAME}
+    exit $error
+}
+trap "cleanup" EXIT TERM INT
+
+mkfs.ext4 -F -j -L root /dev/mapper/${LOOP_DEVICE}
+# No fsck because of X days without checks
+tune2fs -i 0 /dev/mapper/${LOOP_DEVICE}
+
+MOUNT_DIR=`mktemp -d -t build-debimg.XXXXXX`
+mount -o loop /dev/mapper/${LOOP_DEVICE} ${MOUNT_DIR}
 
 info "syncing rootfs to loopback"
-rsync -a -t -r -S -I -H $rootfs/ $rootfs.img.mount
+rsync -a -t -r -S -I -H $rootfs/ ${MOUNT_DIR}
+
+info "install extlinux"
+mkdir -p ${MOUNT_DIR}/boot/extlinux
+echo "default linux
+timeout 1
+label linux
+kernel /vmlinuz
+append initrd=/initrd.img root=LABEL=root biosdevname=0 net.ifnames=0 console=tty0 console=ttyS0,115200 ro" > ${MOUNT_DIR}/boot/extlinux/extlinux.conf
+rm ${MOUNT_DIR}/extlinux.conf || true
+ln ${MOUNT_DIR}/boot/extlinux/extlinux.conf ${MOUNT_DIR}/extlinux.conf
+extlinux --install ${MOUNT_DIR}/boot/extlinux
+
 
 info "umount loopback"
-umount -d $rootfs.img.mount
-rmdir $rootfs.img.mount
+umount -d ${MOUNT_DIR}
+rmdir ${MOUNT_DIR}
 
-info "setting up image directory"
-mkdir $name
-mv $rootfs.img $name/$name.img
-cp $rootfs/boot/vmlinuz-* $name/$name-kernel
-cp $rootfs/boot/initrd.img-* $name/$name-initrd
+fsck.ext3 -f /dev/mapper/${LOOP_DEVICE} || true
 
-info "creating $name-openstack.tar.gz"
-tar --sparse -zcvf $name-openstack.tar.gz $name
+#the next command failed once, so
+sync
+kpartx -d ${AMI_NAME}
 
 if [ -z "$BT_DEBUG" ]; then
     info "removing directory"
     rm -rf $name
 fi
 
+info "creating qcow2 image"
+QCOW2_NAME=$name-openstack.qcow2
+QEMU_VERSION=`qemu-img --help | head -n 1 | cut -d" " -f3 | cut -d"," -f1`
+if dpkg --compare-versions ${QEMU_VERSION} gt 1.0 ; then
+    OTHER_QEMU_IMG_OPTIONS=" -o compat=0.10"
+else
+    OTHER_QEMU_IMG_OPTIONS=""
+fi
+
+qemu-img convert -f raw ${AMI_NAME}${OTHER_QEMU_IMG_OPTIONS} -O qcow2 ${QCOW2_NAME}

bin/openstack-bundle-ami

@@ -0,0 +1,74 @@
+#!/bin/bash -e
+# Copyright (c) 2011-2015 TurnKey GNU/Linux - http://www.turnkeylinux.org
+# 
+# This file is part of buildtasks.
+# 
+# Buildtasks is free software; you can redistribute it and/or modify it
+# under the terms of the GNU Affero General Public License as published by the
+# Free Software Foundation; either version 3 of the License, or (at your
+# option) any later version.
+
+
+fatal() { echo "FATAL [$(basename $0)]: $@" 1>&2; exit 1; }
+info() { echo "INFO [$(basename $0)]: $@"; }
+
+usage() {
+cat<<EOF
+Syntax: $0 rootfs
+Bundles rootfs into an openstack tarball
+
+Arguments::
+
+    rootfs          - root filesystem path
+
+EOF
+exit 1
+}
+
+if [[ "$#" != "1" ]]; then
+    usage
+fi
+
+rootfs=$1
+name=$(echo $rootfs | sed 's/.rootfs//')
+appname=$(echo $name |sed 's/turnkey-\(.*\)-[0-9].*/\1/')
+
+case "$appname" in
+    canvas)           loopsize_padding=524288 ;;
+    ejabberd)         loopsize_padding=524288 ;;
+    appengine-python) loopsize_padding=524288 ;;
+    *)                loopsize_padding=262144 ;;
+esac
+
+info "getting size for loopback"
+rootsize=$(du -s $rootfs | awk '{print $1}')
+loopsize=$[$rootsize + $loopsize_padding]
+
+info "creating sparse loopback"
+dd if=/dev/null of=$rootfs.img bs=1 seek=${loopsize}K
+mkfs.ext4 -F -j $rootfs.img
+
+mkdir $rootfs.img.mount
+mount -o loop $rootfs.img $rootfs.img.mount
+
+info "syncing rootfs to loopback"
+rsync -a -t -r -S -I -H $rootfs/ $rootfs.img.mount
+
+info "umount loopback"
+umount -d $rootfs.img.mount
+rmdir $rootfs.img.mount
+
+info "setting up image directory"
+mkdir $name
+mv $rootfs.img $name/$name.img
+cp $rootfs/boot/vmlinuz-* $name/$name-kernel
+cp $rootfs/boot/initrd.img-* $name/$name-initrd
+
+info "creating $name-openstack.tar.gz"
+tar --sparse -zcvf $name-openstack.tar.gz $name
+
+if [ -z "$BT_DEBUG" ]; then
+    info "removing directory"
+    rm -rf $name
+fi
+

bin/vm-bundle

@@ -77,8 +77,8 @@ kpartx -as $loopdev
 
 pvcreate /dev/mapper/$(basename $loopdev)p2
 vgcreate $VG /dev/mapper/$(basename $loopdev)p2
-lvcreate -L $VM_PARTED_LVM_ROOT -n root $VG
-lvcreate -L $VM_PARTED_LVM_SWAP -n swap_1 $VG
+lvcreate -Zn -L $VM_PARTED_LVM_ROOT -n root $VG
+lvcreate -Zn -L $VM_PARTED_LVM_SWAP -n swap_1 $VG
 vgscan
 vgchange -a y
 

bt-iso

@@ -53,7 +53,7 @@ if [ ! -e $BT_PRODUCTS/$appname ]; then
     cd $BT_PRODUCTS/$appname
 else
     cd $BT_PRODUCTS/$appname
-    git pull
+    #git pull
 fi
 
 deck -D build/root.sandbox || true

bt-openstack

@@ -73,17 +73,17 @@ $BT/bin/aptconf-tag $rootfs openstack
 
 $BT/bin/openstack-bundle $rootfs
 
-$BT/bin/generate-signature $O/$name-openstack.tar.gz
+$BT/bin/generate-signature $O/$name-openstack.qcow2
 
-$BT/bin/generate-buildenv openstack $BT_ISOS/$isofile.sig > $O/$name-openstack.tar.gz.buildenv
+$BT/bin/generate-buildenv openstack $BT_ISOS/$isofile.sig > $O/$name-openstack.qcow2.buildenv
 
 # publish if specified
 if [ "$publish" == "yes" ]; then
     export PUBLISH_DEST=${BT_PUBLISH_IMGS}/openstack/
-    $BT/bin/publish-files $O/$name-openstack.tar.gz
+    $BT/bin/publish-files $O/$name-openstack.qcow2
 
     export PUBLISH_DEST=${BT_PUBLISH_META}/
-    $BT/bin/publish-files $O/$name-openstack.{tar.gz.sig,tar.gz.buildenv}
+    $BT/bin/publish-files $O/$name-openstack.{qcow2.sig,qcow2.buildenv}
 fi
 
 if [ -z "$BT_DEBUG" ] && ! (mount | grep -q $(basename $rootfs)); then

bt-openstack-ami

@@ -0,0 +1,93 @@
+#!/bin/bash -e
+# Copyright (c) 2011-2015 TurnKey GNU/Linux - http://www.turnkeylinux.org
+# 
+# This file is part of buildtasks.
+# 
+# Buildtasks is free software; you can redistribute it and/or modify it
+# under the terms of the GNU Affero General Public License as published by the
+# Free Software Foundation; either version 3 of the License, or (at your
+# option) any later version.
+
+
+fatal() { echo "FATAL [$(basename $0)]: $@" 1>&2; exit 1; }
+warning() { echo "WARNING [$(basename $0)]: $@"; }
+info() { echo "INFO [$(basename $0)]: $@"; }
+
+usage() {
+cat<<EOF
+Syntax: $(basename $0) [--publish] appname
+Converts appliance appname (e.g., core) to openstack image
+
+Options::
+
+    --publish           - if set, image will be devpay'ed and made public
+
+Environment::
+
+    BT_DEBUG            - turn on debugging
+EOF
+exit 1
+}
+
+while [ "$1" != "" ]; do
+    case $1 in
+        --help|-h )  usage;;
+        --publish)   publish="yes";;
+        *)           if [ -n "$appname" ]; then usage; else appname=$1; fi ;;
+    esac
+    shift
+done
+
+[ -n "$appname" ] || usage
+[ -n "$publish" ] || warning "--publish was not specified"
+
+[ -n "$BT_DEBUG" ] && set -x
+
+export BT=$(dirname $(readlink -f $0))
+export BT_CONFIG=$BT/config
+. $BT_CONFIG/common.cfg
+. $BT_CONFIG/build.cfg
+
+O=$BT_BUILDS/openstack
+mkdir -p $O
+
+[ -n "$BT_VERSION" ] || fatal "BT_VERSION not set"
+
+isofile=turnkey-$appname-$BT_VERSION.iso
+name=turnkey-$appname-$BT_VERSION
+rootfs=$name.rootfs
+cdroot=$name.cdroot
+
+$BT/bin/iso-download $BT_ISOS $BT_VERSION $appname
+$BT/bin/iso-verify $BT_ISOS $BT_VERSION $appname
+
+cd $O
+tklpatch-extract-iso $BT_ISOS/$isofile
+
+tklpatch-apply $rootfs $BT/patches/headless
+tklpatch-apply $rootfs $BT/patches/cloud
+tklpatch-apply $rootfs $BT/patches/openstack-ami
+$BT/bin/rootfs-cleanup $rootfs
+
+$BT/bin/aptconf-tag $rootfs openstack
+
+$BT/bin/openstack-bundle-ami $rootfs
+
+$BT/bin/generate-signature $O/$name-openstack.tar.gz
+
+$BT/bin/generate-buildenv openstack $BT_ISOS/$isofile.sig > $O/$name-openstack.tar.gz.buildenv
+
+# publish if specified
+if [ "$publish" == "yes" ]; then
+    export PUBLISH_DEST=${BT_PUBLISH_IMGS}/openstack/
+    $BT/bin/publish-files $O/$name-openstack.tar.gz
+
+    export PUBLISH_DEST=${BT_PUBLISH_META}/
+    $BT/bin/publish-files $O/$name-openstack.{tar.gz.sig,tar.gz.buildenv}
+fi
+
+if [ -z "$BT_DEBUG" ] && ! (mount | grep -q $(basename $rootfs)); then
+    rm -rf $rootfs
+    rm -rf $cdroot
+fi
+

docs/setup

@@ -22,6 +22,11 @@ bt-ec2 ::
     apt-get install parted ec2metadata
     pip install boto
 
+bt-openstack ::
+
+    apt-get install parted ec2metadata mbr qemu kpartx extlinux
+    pip install boto
+
 bt-vm ::
 
     # vmware ovftool

patches/cloud/conf

@@ -20,4 +20,3 @@ update-rc.d -f confconsole disable
 
 # redirect inithook output (preseeded headless deployment)
 sed -i '/REDIRECT_OUTPUT/ s/=.*/=true/g' /etc/default/inithooks
-