Secure upgrade (#2337)

#### What I did
Added support for secure upgrade

#### How I did it
It includes image signing during build (in sonic buildimage repo) and verification during image install (in sonic-utilities).
HLD can be found in the following PR: sonic-net/SONiC#1024
#### How to verify it
Feature is used to allow image was not modified since built from vendor. During installation, image can be verified with a signature attached to it.
In order for image verification - image must be signed - need to provide signing key and certificate (paths in SECURE_UPGRADE_DEV_SIGNING_KEY and SECURE_UPGRADE_DEV_SIGNING_CERT in rules/config) during build , and during image install, need to enable secure boot flag in bios, and signing_certificate should be available in bios.

#### Feature dependencies
In order for this feature to work smoothly, need to have secure boot feature implemented as well.
The Secure boot feature will be merged in the near future.

sonic-buildimage PR: sonic-net/sonic-buildimage#11862

scripts/verify_image_sign.sh

@@ -0,0 +1,75 @@
+#!/bin/sh
+image_file="${1}"
+cms_sig_file="sig.cms"
+lines_for_lookup=50
+SECURE_UPGRADE_ENABLED=0
+DIR="$(dirname "$0")"
+if [ -d "/sys/firmware/efi/efivars" ]; then
+    if ! [ -n "$(ls -A /sys/firmware/efi/efivars 2>/dev/null)" ]; then
+        mount -t efivarfs none /sys/firmware/efi/efivars 2>/dev/null
+    fi
+    SECURE_UPGRADE_ENABLED=$(bootctl status 2>/dev/null | grep -c "Secure Boot: enabled")
+else
+    echo "efi not supported - exiting without verification"
+    exit 0
+fi
+
+. /usr/local/bin/verify_image_sign_common.sh
+
+if [ ${SECURE_UPGRADE_ENABLED} -eq 0 ]; then
+    echo "secure boot not enabled - exiting without image verification"
+    exit 0
+fi
+
+clean_up ()
+{
+    if  [ -d ${EFI_CERTS_DIR} ]; then rm -rf ${EFI_CERTS_DIR}; fi
+    if  [ -d "${TMP_DIR}" ]; then rm -rf ${TMP_DIR}; fi
+    exit $1
+}
+
+TMP_DIR=$(mktemp -d)
+DATA_FILE="${TMP_DIR}/data.bin"
+CMS_SIG_FILE="${TMP_DIR}/${cms_sig_file}"
+TAR_SIZE=$(head -n $lines_for_lookup $image_file | grep "payload_image_size=" | cut -d"=" -f2- )
+SHARCH_SIZE=$(sed '/^exit_marker$/q' $image_file | wc -c)
+SIG_PAYLOAD_SIZE=$(($TAR_SIZE + $SHARCH_SIZE ))
+# Extract cms signature from signed file
+# Add extra byte for payload
+sed -e '1,/^exit_marker$/d' $image_file | tail -c +$(( $TAR_SIZE + 1 )) > $CMS_SIG_FILE
+# Extract image from signed file
+head -c $SIG_PAYLOAD_SIZE $image_file > $DATA_FILE
+# verify signature with certificate fetched with efi tools
+EFI_CERTS_DIR=/tmp/efi_certs
+[ -d $EFI_CERTS_DIR ] && rm  -rf $EFI_CERTS_DIR
+mkdir $EFI_CERTS_DIR
+efi-readvar -v db -o $EFI_CERTS_DIR/db_efi >/dev/null || 
+{
+    echo "Error: unable to read certs from efi db: $?"
+    clean_up 1
+}
+# Convert one file to der certificates
+sig-list-to-certs $EFI_CERTS_DIR/db_efi $EFI_CERTS_DIR/db >/dev/null||
+{
+    echo "Error: convert sig list to certs: $?"
+    clean_up 1
+}
+for file in $(ls $EFI_CERTS_DIR | grep "db-"); do
+    LOG=$(openssl  x509 -in $EFI_CERTS_DIR/$file -inform der -out $EFI_CERTS_DIR/cert.pem 2>&1)
+    if [ $? -ne 0 ]; then
+        logger "cms_validation: $LOG"
+    fi
+    # Verify detached signature
+    LOG=$(verify_image_sign_common $image_file $DATA_FILE $CMS_SIG_FILE)
+    VALIDATION_RES=$?
+    if [ $VALIDATION_RES -eq 0 ]; then
+        RESULT="CMS Verified OK using efi keys"
+        echo "verification ok:$RESULT"
+        # No need to continue.
+        # Exit without error if any success signature verification.
+        clean_up 0
+    fi
+done
+echo "Failure: CMS signature Verification Failed: $LOG"
+
+clean_up 1

scripts/verify_image_sign_common.sh

@@ -0,0 +1,34 @@
+#!/bin/bash
+verify_image_sign_common() {
+    image_file="${1}"
+    cms_sig_file="sig.cms"
+    TMP_DIR=$(mktemp -d)
+    DATA_FILE="${2}"
+    CMS_SIG_FILE="${3}"
+
+    openssl version | awk '$2 ~ /(^0\.)|(^1\.(0\.|1\.0))/ { exit 1 }'
+    if [ $? -eq 0 ]; then
+        # for version 1.1.1 and later
+        no_check_time="-no_check_time"
+    else
+        # for version older than 1.1.1 use noattr
+        no_check_time="-noattr"
+    fi
+
+    # making sure image verification is supported
+    EFI_CERTS_DIR=/tmp/efi_certs
+    RESULT="CMS Verification Failure"
+    LOG=$(openssl cms -verify $no_check_time -noout -CAfile $EFI_CERTS_DIR/cert.pem -binary -in ${CMS_SIG_FILE} -content ${DATA_FILE} -inform pem 2>&1 > /dev/null )
+    VALIDATION_RES=$?
+    if [ $VALIDATION_RES -eq 0 ]; then
+        RESULT="CMS Verified OK"
+        if  [ -d "${TMP_DIR}" ]; then rm -rf ${TMP_DIR}; fi
+        echo "verification ok:$RESULT"
+        # No need to continue.
+        # Exit without error if any success signature verification.
+        return 0
+    fi
+
+    if  [ -d "${TMP_DIR}" ]; then rm -rf ${TMP_DIR}; fi
+    return 1
+}

setup.py

@@ -154,6 +154,8 @@
         'scripts/memory_threshold_check_handler.py',
         'scripts/techsupport_cleanup.py',
         'scripts/storm_control.py',
+        'scripts/verify_image_sign.sh',
+        'scripts/verify_image_sign_common.sh',
         'scripts/check_db_integrity.py',
         'scripts/sysreadyshow'
     ],

sonic_installer/bootloader/grub.py

@@ -153,6 +153,17 @@ def verify_image_platform(self, image_path):
         # Check if platform is inside image's target platforms
         return self.platform_in_platforms_asic(platform, image_path)
 
+    def verify_image_sign(self, image_path):
+        click.echo('Verifying image signature')
+        verification_script_name = 'verify_image_sign.sh'
+        script_path = os.path.join('/usr', 'local', 'bin', verification_script_name)
+        if not os.path.exists(script_path):
+            click.echo("Unable to find verification script in path " + script_path)
+            return False
+        verification_result = subprocess.run([script_path, image_path], capture_output=True)
+        click.echo(str(verification_result.stdout) + " " + str(verification_result.stderr))
+        return verification_result.returncode == 0
+
     @classmethod
     def detect(cls):
         return os.path.isfile(os.path.join(HOST_PATH, 'grub/grub.cfg'))

sonic_installer/main.py

@@ -511,7 +511,8 @@ def sonic_installer():
 @click.option('-y', '--yes', is_flag=True, callback=abort_if_false,
               expose_value=False, prompt='New image will be installed, continue?')
 @click.option('-f', '--force', '--skip-secure-check', is_flag=True,
-              help="Force installation of an image of a non-secure type than secure running image")
+              help="Force installation of an image of a non-secure type than secure running " +
+              " image, this flag does not affect secure upgrade image verification")
 @click.option('--skip-platform-check', is_flag=True,
               help="Force installation of an image of a type which is not of the same platform")
 @click.option('--skip_migration', is_flag=True,
@@ -576,6 +577,14 @@ def install(url, force, skip_platform_check=False, skip_migration=False, skip_pa
                 "Aborting...", LOG_ERR)
             raise click.Abort()
 
+        # Calling verification script by default - signature will be checked if enabled in bios
+        echo_and_log("Verifing image {} signature...".format(binary_image_version))
+        if not bootloader.verify_image_sign(image_path):
+            echo_and_log('Error: Failed verify image signature', LOG_ERR)
+            raise click.Abort()
+        else:
+            echo_and_log('Verification successful')
+
         echo_and_log("Installing image {} and setting it as default...".format(binary_image_version))
         with SWAPAllocator(not skip_setup_swap, swap_mem_size, total_mem_threshold, available_mem_threshold):
             bootloader.install_image(image_path)
@@ -958,5 +967,6 @@ def verify_next_image():
         sys.exit(1)
     click.echo('Image successfully verified')
 
+
 if __name__ == '__main__':
     sonic_installer()

tests/installer_bootloader_grub_test.py

@@ -53,3 +53,11 @@ def test_set_fips_grub():
 
     # Cleanup the _tmp_host folder
     shutil.rmtree(tmp_host_path)
+
+def test_verify_image():
+
+    bootloader = grub.GrubBootloader()
+    image = f'{grub.IMAGE_PREFIX}expeliarmus-{grub.IMAGE_PREFIX}abcde'
+
+    # command should fail
+    assert not bootloader.verify_image_sign(image)

tests/scripts/create_mock_image.sh

@@ -0,0 +1,40 @@
+repo_dir=$1
+input_image=$2
+output_file=$3
+cert_file=$4
+key_file=$5
+tmp_dir=
+clean_up()
+{
+    sudo rm -rf $tmp_dir
+    sudo rm -rf $output_file
+    exit $1
+}
+
+DIR="$(dirname "$0")"
+
+tmp_dir=$(mktemp -d)
+sha1=$(cat $input_image | sha1sum | awk '{print $1}')
+echo -n "."
+cp $repo_dir/installer/sharch_body.sh $output_file || {
+    echo "Error: Problems copying sharch_body.sh"
+    clean_up 1
+}
+# Replace variables in the sharch template
+sed -i -e "s/%%IMAGE_SHA1%%/$sha1/" $output_file
+echo -n "."
+tar_size="$(wc -c < "${input_image}")"
+cat $input_image >> $output_file
+sed -i -e "s|%%PAYLOAD_IMAGE_SIZE%%|${tar_size}|" ${output_file}
+CMS_SIG="${tmp_dir}/signature.sig"
+
+echo "$0 CMS signing ${input_image} with ${key_file}. Output file ${output_file}"
+. $repo_dir/scripts/sign_image_dev.sh
+sign_image_dev  ${cert_file} ${key_file} $output_file ${CMS_SIG} || clean_up 1
+
+cat ${CMS_SIG} >> ${output_file}
+echo "Signature done."
+# append signature to binary
+sudo rm -rf ${CMS_SIG}
+sudo rm -rf $tmp_dir
+exit 0

tests/scripts/create_sign_and_verify_test_files.sh

@@ -0,0 +1,91 @@
+repo_dir=$1
+out_dir=$2
+mock_image="mock_img.bin"
+output_file=$out_dir/output_file.bin
+cert_file=$3
+other_cert_file=$4
+tmp_dir=
+clean_up()
+{
+    sudo rm -rf $tmp_dir
+    sudo rm -rf $mock_image
+    exit $1
+}
+DIR="$(dirname "$0")"
+[ -d $out_dir ] || rm -rf $out_dir
+mkdir $out_dir
+tmp_dir=$(mktemp -d)
+#generate self signed keys and certificate
+key_file=$tmp_dir/private-key.pem
+pub_key_file=$tmp_dir/public-key.pem
+openssl ecparam -name secp256r1 -genkey -noout -out $key_file
+openssl ec -in $key_file -pubout -out $pub_key_file
+openssl req -new -x509 -key $key_file -out $cert_file -days 360 -subj "/C=US/ST=Test/L=Test/O=Test/CN=Test"
+alt_key_file=$tmp_dir/alt-private-key.pem
+alt_pub_key_file=$tmp_dir/alt-public-key.pem
+openssl ecparam -name secp256r1 -genkey -noout -out $alt_key_file
+openssl ec -in $alt_key_file -pubout -out $alt_pub_key_file
+openssl req -new -x509 -key $alt_key_file -out $other_cert_file -days 360 -subj "/C=US/ST=Test/L=Test/O=Test/CN=Test"
+
+echo "this is a mock image\nThis is another line !2#4%6\n" > $mock_image
+echo "Created a mock image with following text:"
+cat $mock_image
+# create signed mock image
+
+sh $DIR/create_mock_image.sh $repo_dir $mock_image $output_file $cert_file $key_file || {
+    echo "Error: unable to create mock image"   
+    clean_up 1
+}
+
+[ -f "$output_file" ] || {
+    echo "signed mock image not created - exiting without testing"
+    clean_up 1
+}
+
+test_image_1=$out_dir/test_image_1.bin
+cp -v $output_file $test_image_1 || {
+    echo "Error: Problems copying image"
+    clean_up 1
+}
+
+# test_image_1 = modified image size to something else - should fail on signature verification
+image_size=$(sed -n 's/^payload_image_size=\(.*\)/\1/p' < $test_image_1)
+sed -i "/payload_image_size=/c\payload_image_size=$(($image_size - 5))" $test_image_1
+
+test_image_2=$out_dir/test_image_2.bin
+cp -v $output_file $test_image_2 || {
+    echo "Error: Problems copying image"
+    clean_up 1
+}
+
+# test_image_2 = modified image sha1 to other sha1 value - should fail on signature verification
+im_sha=$(sed -n 's/^payload_sha1=\(.*\)/\1/p' < $test_image_2)
+sed -i "/payload_sha1=/c\payload_sha1=2f1bbd5a0d411253103e688e4e66c00c94bedd40" $test_image_2
+
+tmp_image=$tmp_dir/"tmp_image.bin"
+echo "this is a different image now" >> $mock_image
+sh $DIR/create_mock_image.sh $repo_dir $mock_image $tmp_image $cert_file $key_file || {
+    echo "Error: unable to create mock image"   
+    clean_up 1
+}
+# test_image_3 = original mock image with wrong signature
+# Extract cms signature from signed file
+test_image_3=$out_dir/"test_image_3.bin"
+tmp_sig="${tmp_dir}/tmp_sig.sig"
+TMP_TAR_SIZE=$(head -n 50 $tmp_image | grep "payload_image_size=" | cut -d"=" -f2- )
+sed -e '1,/^exit_marker$/d' $tmp_image | tail -c +$(( $TMP_TAR_SIZE + 1 )) > $tmp_sig
+
+TAR_SIZE=$(head -n 50 $output_file | grep "payload_image_size=" | cut -d"=" -f2- )
+SHARCH_SIZE=$(sed '/^exit_marker$/q' $output_file | wc -c)
+SIG_PAYLOAD_SIZE=$(($TAR_SIZE + $SHARCH_SIZE ))
+head -c $SIG_PAYLOAD_SIZE $output_file > $test_image_3
+sudo rm -rf $tmp_image
+
+cat ${tmp_sig} >> ${test_image_3}
+
+# test_image_4 = modified image with original mock image signature
+test_image_4=$out_dir/"test_image_4.bin"
+head -c $SIG_PAYLOAD_SIZE $output_file > $test_image_4
+echo "this is additional line" >> $test_image_4
+cat ${tmp_sig} >> ${test_image_4}
+clean_up 0

tests/scripts/verify_image_sign_test.sh

@@ -0,0 +1,29 @@
+#!/bin/bash
+image_file="${1}"
+cert_path="${2}"
+cms_sig_file="sig.cms"
+TMP_DIR=$(mktemp -d)
+DATA_FILE="${TMP_DIR}/data.bin"
+CMS_SIG_FILE="${TMP_DIR}/${cms_sig_file}"
+lines_for_lookup=50
+
+TAR_SIZE=$(head -n $lines_for_lookup $image_file | grep "payload_image_size=" | cut -d"=" -f2- )
+SHARCH_SIZE=$(sed '/^exit_marker$/q' $image_file | wc -c)
+SIG_PAYLOAD_SIZE=$(($TAR_SIZE + $SHARCH_SIZE ))
+# Extract cms signature from signed file - exit marker marks last sharch prefix + number of image lines + 1 for next linel
+# Add extra byte for payload - extracting image signature from line after data file
+sed -e '1,/^exit_marker$/d' $image_file | tail -c +$(( $TAR_SIZE + 1 )) > $CMS_SIG_FILE
+# Extract image from signed file
+head -c $SIG_PAYLOAD_SIZE $image_file > $DATA_FILE
+EFI_CERTS_DIR=/tmp/efi_certs
+[ -d $EFI_CERTS_DIR ] && rm  -rf $EFI_CERTS_DIR
+mkdir $EFI_CERTS_DIR
+cp $cert_path $EFI_CERTS_DIR/cert.pem
+
+DIR="$(dirname "$0")"
+. $DIR/verify_image_sign_common.sh
+verify_image_sign_common $image_file $DATA_FILE $CMS_SIG_FILE
+VERIFICATION_RES=$?
+if  [ -d "${TMP_DIR}" ]; then rm -rf ${TMP_DIR}; fi
+[ -d $EFI_CERTS_DIR ] && rm  -rf $EFI_CERTS_DIR
+exit $VERIFICATION_RES