From 314cc502ac862494973712537bf7a7a0c34f96d8 Mon Sep 17 00:00:00 2001 From: Arif Ali Date: Tue, 7 Nov 2023 13:58:25 +0200 Subject: [PATCH] [debian] Packaging and testing for debian pkgs Initial start of building and testing the deb package This will improve the overall testing of sos over the Ubuntu releases, and also helps with the basis of thenadding further stagetwo testing in the future. This will help to pick up issues automatically rather than the manual testing. Add ubuntu-latest for deb CI build and testing. Update the GCE images for all ubuntu images for CI testing Co-authored-by: Nikhil Kshirsagar Signed-off-by: Nikhil Kshirsagar Signed-off-by: Arif Ali --- .cirrus.yml | 59 +++- .gitignore | 6 + debian/changelog | 5 + debian/control | 28 ++ debian/copyright | 23 ++ debian/dirs | 5 + debian/docs | 1 + debian/install | 1 + .../0001-debian-change-tmp-dir-location.patch | 11 + debian/patches/series | 1 + debian/rules | 15 + debian/source/format | 1 + debian/tests/control | 3 + debian/tests/simple.sh | 272 ++++++++++++++++++ debian/watch | 4 + 15 files changed, 426 insertions(+), 9 deletions(-) create mode 100644 debian/changelog create mode 100644 debian/control create mode 100644 debian/copyright create mode 100644 debian/dirs create mode 100644 debian/docs create mode 100644 debian/install create mode 100644 debian/patches/0001-debian-change-tmp-dir-location.patch create mode 100644 debian/patches/series create mode 100755 debian/rules create mode 100644 debian/source/format create mode 100644 debian/tests/control create mode 100755 debian/tests/simple.sh create mode 100644 debian/watch diff --git a/.cirrus.yml b/.cirrus.yml index acff84333c..0971c84f0a 100644 --- a/.cirrus.yml +++ b/.cirrus.yml @@ -8,6 +8,7 @@ env: DEBIAN_NAME: "debian-11" + UBUNTU_LATEST_NAME: "ubuntu-23.10" UBUNTU_NAME: "ubuntu-22.04" UBUNTU_PRIOR_NAME: "ubuntu-20.04" UBUNTU_PRIOR2_NAME: "ubuntu-18.04" @@ -27,10 +28,11 @@ env: DEBIAN_IMAGE_NAME: "debian-11-bullseye-v20230809" FEDORA_IMAGE_NAME: "fedora-cloud-base-gcp-38-1-6-x86-64" FEDORA_PRIOR_IMAGE_NAME: "fedora-cloud-base-gcp-37-1-7-x86-64" - UBUNTU_IMAGE_NAME: "ubuntu-2204-jammy-v20230727" - UBUNTU_PRIOR_IMAGE_NAME: "ubuntu-2004-focal-v20230724" + UBUNTU_LATEST_IMAGE_NAME: "ubuntu-2310-mantic-amd64-v20231031" + UBUNTU_IMAGE_NAME: "ubuntu-2204-jammy-v20231030" + UBUNTU_PRIOR_IMAGE_NAME: "ubuntu-2004-focal-v20231101" UBUNTU_PRIOR2_IMAGE_NAME: "ubuntu-1804-bionic-v20230605" - UBUNTU_SNAP_IMAGE_NAME: "ubuntu-2204-jammy-v20230727" + UBUNTU_SNAP_IMAGE_NAME: "ubuntu-2204-jammy-v20231030" # Curl-command prefix for downloading task artifacts, simply add the # the url-encoded task name, artifact name, and path as a suffix. @@ -115,6 +117,27 @@ rpm_build_task: path: ./sos_${BUILD_NAME}.rpm type: application/octet-stream +# Make sure a user can manually build a deb from the checkout +deb_build_task: + alias: "deb_build" + name: "deb Build From Checkout" + gce_instance: + image_project: "${UBUNTU_PROJECT}" + image_name: "${UBUNTU_LATEST_IMAGE_NAME}" + type: e2-medium + setup_script: | + apt update --allow-releaseinfo-change + apt -y install devscripts equivs python3-pip + mk-build-deps + apt -y install ./sosreport-build-deps*.deb + pip3 install avocado-framework==94.0 --break-system-packages + main_script: | + dpkg-buildpackage -b -us -uc -rfakeroot -m --build-by="noreply@canonical.com" + prep_artifacts_script: mv ../*.deb ./sos_cirrus.deb + packages_artifacts: + path: ./sos_cirrus.deb + type: application/octet-stream + # Make sure a user can manually build a snap from the checkout snap_build_task: alias: "snap_build" @@ -146,6 +169,7 @@ report_stageone_task: depends_on: - rpm_build - snap_build + - deb_build gce_instance: *standardvm matrix: - env: *centos9 @@ -156,24 +180,38 @@ report_stageone_task: PROJECT: ${UBUNTU_PROJECT} BUILD_NAME: ${UBUNTU_NAME} VM_IMAGE_NAME: ${UBUNTU_IMAGE_NAME} + PKG: "snap" - env: &ubuntuprior PROJECT: ${UBUNTU_PROJECT} BUILD_NAME: ${UBUNTU_PRIOR_NAME} VM_IMAGE_NAME: ${UBUNTU_PRIOR_IMAGE_NAME} + PKG: "snap" - env: &ubuntuprior2 PROJECT: ${UBUNTU_PROJECT} BUILD_NAME: ${UBUNTU_PRIOR2_NAME} VM_IMAGE_NAME: ${UBUNTU_PRIOR2_IMAGE_NAME} + PKG: "snap" + - env: &ubuntu-latest + PROJECT: ${UBUNTU_PROJECT} + BUILD_NAME: ${UBUNTU_LATEST_NAME} + VM_IMAGE_NAME: ${UBUNTU_LATEST_IMAGE_NAME} + PKG: "deb" setup_script: &setup | if [ $(command -v apt) ]; then - echo "$ARTCURL/snap%20Build%20From%20Checkout/packages/sosreport_test_amd64.snap" - $ARTCURL/snap%20Build%20From%20Checkout/packages/sosreport_test_amd64.snap apt -y purge sosreport apt update --allow-releaseinfo-change apt -y install python3-pip snapd - systemctl start snapd - snap install ./sosreport_test_amd64.snap --classic --dangerous - snap alias sosreport.sos sos + if [ ${PKG} == "snap" ] ; then + echo "$ARTCURL/snap%20Build%20From%20Checkout/packages/sosreport_test_amd64.snap" + $ARTCURL/snap%20Build%20From%20Checkout/packages/sosreport_test_amd64.snap + systemctl start snapd + snap install ./sosreport_test_amd64.snap --classic --dangerous + snap alias sosreport.sos sos + elif [ ${PKG} == "deb" ]; then + echo "$ARTCURL/deb%20Build%20From%20Checkout/packages/sos_cirrus.deb" + $ARTCURL/deb%20Build%20From%20Checkout/packages/sos_cirrus.deb + apt -y install ./sos_cirrus.deb + fi fi if [ $(command -v dnf) ]; then echo "$ARTCURL/rpm%20Build%20From%20Checkout%20-%20${BUILD_NAME}/packages/sos_${BUILD_NAME}.rpm" @@ -182,7 +220,9 @@ report_stageone_task: dnf -y install python3-pip ethtool dnf -y install ./sos_${BUILD_NAME}.rpm fi - pip3 install avocado-framework==94.0 + PIP_EXTRA="" + [[ $(pip3 install --help | grep break-system) ]] && PIP_EXTRA="--break-system-packages" + pip3 install avocado-framework==94.0 ${PIP_EXTRA} # run the unittests separately as they require a different PYTHONPATH in # order for the imports to work properly under avocado unittest_script: PYTHONPATH=. avocado run tests/unittests/ @@ -205,6 +245,7 @@ report_stagetwo_task: - env: *centos8 - env: *fedora - env: *ubuntu + - env: *ubuntu-latest setup_script: *setup install_pexpect_script: | if [ $(command -v apt) ]; then diff --git a/.gitignore b/.gitignore index 318b19e6d6..cbe9c61399 100644 --- a/.gitignore +++ b/.gitignore @@ -22,3 +22,9 @@ docs/_build # Pycharm .idea/ + +# debian files +debian/sosreport* +debian/files +debian/.debhelper +debian/debhelper-build-stamp diff --git a/debian/changelog b/debian/changelog new file mode 100644 index 0000000000..d6aadef47f --- /dev/null +++ b/debian/changelog @@ -0,0 +1,5 @@ +sosreport (4.6.0-0ubuntu1) mantic; urgency=medium + + * New 4.6.0 upstream. + + -- Nikhil Kshirsagar Thu, 17 Aug 2023 08:17:20 +0000 diff --git a/debian/control b/debian/control new file mode 100644 index 0000000000..6542e97d69 --- /dev/null +++ b/debian/control @@ -0,0 +1,28 @@ +Source: sosreport +Maintainer: Nikhil Kshirsagar +Section: admin +Priority: optional +Standards-Version: 4.5.1 +Build-Depends: + debhelper-compat (= 13), + dh-python, + gettext, + python3-all, + python3-coverage, + python3-nose, + python3-setuptools, + python3-sphinx, + python3-magic, + python3-pexpect, +Homepage: https://github.com/sosreport/sos +Vcs-Browser: https://salsa.debian.org/sosreport-team/sosreport +Vcs-Git: https://salsa.debian.org/sosreport-team/sosreport.git + +Package: sosreport +Architecture: any +Depends: ${python3:Depends}, ${misc:Depends}, python3-pexpect, python3-magic +Description: Set of tools to gather troubleshooting data from a system + Sos is a set of tools that gathers information about system + hardware and configuration. The information can then be used for + diagnostic purposes and debugging. Sos is commonly used to help + support technicians and developers. diff --git a/debian/copyright b/debian/copyright new file mode 100644 index 0000000000..921c89370d --- /dev/null +++ b/debian/copyright @@ -0,0 +1,23 @@ +Format: https://www.debian.org/doc/packaging-manuals/copyright-format/1.0/ +Upstream-Name: sosreport +Upstream-Contact: Jake Hunsaker +Source: https://github.com/sosreport/sos + +License: GPL-2+ + This program is free software; you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation; either version 2 of the License, or (at + your option) any later version. + . + This program is distributed in the hope that it will be useful, but + WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + General Public License for more details. + . + You should have received a copy of the GNU General Public License + along with this program; if not, write to the Free Software + Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301, + USA. + . + On Debian systems, the complete text of the GNU General Public + License, version 2, can be found in /usr/share/common-licenses/GPL-2. diff --git a/debian/dirs b/debian/dirs new file mode 100644 index 0000000000..78757791d7 --- /dev/null +++ b/debian/dirs @@ -0,0 +1,5 @@ +/etc/sos +/etc/sos/cleaner +/etc/sos/extras.d +/etc/sos/groups.d +/etc/sos/presets.d diff --git a/debian/docs b/debian/docs new file mode 100644 index 0000000000..475d728a0e --- /dev/null +++ b/debian/docs @@ -0,0 +1 @@ +README.md AUTHORS diff --git a/debian/install b/debian/install new file mode 100644 index 0000000000..a54b194084 --- /dev/null +++ b/debian/install @@ -0,0 +1 @@ +sos.conf etc/sos/ diff --git a/debian/patches/0001-debian-change-tmp-dir-location.patch b/debian/patches/0001-debian-change-tmp-dir-location.patch new file mode 100644 index 0000000000..29ddefa7d8 --- /dev/null +++ b/debian/patches/0001-debian-change-tmp-dir-location.patch @@ -0,0 +1,11 @@ +Description: Change default location to /tmp +--- a/sos.conf ++++ b/sos.conf +@@ -7,6 +7,7 @@ + #verify = yes + #batch = yes + #log-size = 15 ++tmp-dir = /tmp + + [report] + # Options that will apply to any `sos report` run should be listed here. diff --git a/debian/patches/series b/debian/patches/series new file mode 100644 index 0000000000..3d9fdd4166 --- /dev/null +++ b/debian/patches/series @@ -0,0 +1 @@ +0001-debian-change-tmp-dir-location.patch diff --git a/debian/rules b/debian/rules new file mode 100755 index 0000000000..79b75aad9e --- /dev/null +++ b/debian/rules @@ -0,0 +1,15 @@ +#!/usr/bin/make -f + +export PYBUILD_NAME=sosreport + +%: + dh $@ --with python3 --buildsystem=pybuild + +override_dh_install: + # Move config file to the right location. + mv debian/sosreport/usr/config/sos.conf debian/sosreport/etc/sos/sos.conf + # Remove unnecessary unused dir. + rm -rf debian/sosreport/usr/config + +override_dh_auto_test: + nosetests3 -v --with-cover --cover-package=sos tests/unittests diff --git a/debian/source/format b/debian/source/format new file mode 100644 index 0000000000..163aaf8d82 --- /dev/null +++ b/debian/source/format @@ -0,0 +1 @@ +3.0 (quilt) diff --git a/debian/tests/control b/debian/tests/control new file mode 100644 index 0000000000..77909a088f --- /dev/null +++ b/debian/tests/control @@ -0,0 +1,3 @@ +Tests: simple.sh +Depends: sosreport +Restrictions: needs-root, allow-stderr, isolation-machine diff --git a/debian/tests/simple.sh b/debian/tests/simple.sh new file mode 100755 index 0000000000..8fc9b39ea2 --- /dev/null +++ b/debian/tests/simple.sh @@ -0,0 +1,272 @@ +#!/bin/bash +# This file is part of the sos project: https://github.com/sosreport/sos +# +# This copyrighted material is made available to anyone wishing to use, +# modify, copy, or redistribute it subject to the terms and conditions of +# version 2 of the GNU General Public License. +# +# See the LICENSE file in the source distribution for further information. +# A quick port of the travis tests to bash, requires root +# TODO +# * look into using a framework.. +# * why --dry-run fails? +# * why --experimental fails? +# * make it better validate archives and contents + +PYTHON=${1:-/usr/bin/python3} +SOSPATH=${2:-./bin/sos report --batch --tmp-dir=/tmp } + +NUMOFFAILURES=0 +summary="\nSummary\n" +FAIL_LIST="" + +run_expecting_success () { + #$1 - is command options + #$2 - kind of check to do, so far only extract + FAIL=false + # Make sure clean + rm -f /dev/shm/stderr /dev/shm/stdout /tmp/sosreport*.tar.* + rm -rf /tmp/sosreport_test/ + + start=`date +%s` + echo "######### RUNNING $1 #########" + $PYTHON $SOSPATH $1 2> /dev/shm/stderr 1> /dev/shm/stdout + + if [ $? -eq 0 ]; then + echo "### Success" + else + echo "!!! FAILED !!!" + add_failure "$1 failed during execution" + fi + + end=`date +%s` + runtime=$((end-start)) + echo "#### Sos Total time (seconds):" $runtime + + if [ -s /dev/shm/stderr ]; then + add_failure "test generated stderr output, see above" + echo "### start stderr" + cat /dev/shm/stderr + echo "### end stderr" + fi + + echo "### start stdout" + cat /dev/shm/stdout + echo "### end stdout" + + if [ "extract" = "$2" ]; then + echo "### start extraction" + rm -f /tmp/sosreport*sha256 + mkdir /tmp/sosreport_test/ + tar xfa /tmp/sosreport*.tar* -C /tmp/sosreport_test --strip-components=1 + if [ -s /tmp/sosreport_test/sos_logs/*errors.txt ]; then + FAIL=true + echo "!!! FAILED !!!" + add_failure "Test $1 generated errors" + echo "#### *errors.txt output" + ls -alh /tmp/sosreport_test/sos_logs/ + cat /tmp/sosreport_test/sos_logs/*errors.txt + fi + echo "### stop extraction" + fi + + echo "######### DONE WITH $1 #########" + + if $FAIL; then + NUMOFFAILURES=$(($NUMOFFAILURES + 1)) + return 1 + else + return 0 + fi +} + +update_summary () { + size="$(grep Size /dev/shm/stdout)" + size="$(echo "${size:-"Size 0.00MiB"}")" + summary="${summary} \n failures ${FAIL} \t time ${runtime} \t ${size} \t ${1} " +} + +update_failures () { + if $FAIL; then + NUMOFFAILURES=$(($NUMOFFAILURES + 1)) + fi +} + +add_failure () { + FAIL=true + echo "!!! TEST FAILED: $1 !!!" + FAIL_LIST="${FAIL_LIST}\n \t ${FUNCNAME[1]}: \t\t ${1}" +} + +# Test a no frills run with verbosity and make sure the expected items exist +test_normal_report () { + cmd="-vvv" + # get a list of initial kmods loaded + kmods=( $(lsmod | cut -f1 -d ' ' | sort) ) + run_expecting_success "$cmd" extract + if [ $? -eq 0 ]; then + if [ ! -f /tmp/sosreport_test/sos_reports/sos.html ]; then + add_failure "did not generate html reports" + fi + if [ ! -f /tmp/sosreport_test/sos_reports/manifest.json ]; then + add_failure "did not generate manifest.json" + fi + if [ ! -f /tmp/sosreport_test/free ]; then + add_failure "did not create free symlink in archive root" + fi + if [ ! "$(grep "DEBUG" /tmp/sosreport_test/sos_logs/sos.log)" ]; then + add_failure "did not find debug logging when using -vvv" + fi + # new list, see if we added any + new_kmods=( $(lsmod | cut -f1 -d ' ' | sort) ) + if [ "$(printf '%s\n' "${kmods[@]}" "${new_kmods[@]}" | sort | uniq -u)" ]; then + add_failure "new kernel modules loaded during execution" + echo "$(printf '%s\n' "${kmods[@]}" "${new_kmods[@]}" | sort | uniq -u)" + fi + update_failures + update_summary "$cmd" + fi +} + +# Test for correctly skipping html generation, and label setting +test_noreport_label_only () { + cmd="--no-report --label TEST -o hardware" + run_expecting_success "$cmd" extract + if [ $? -eq 0 ]; then + if [ -f /tmp/sosreport_test/sos_reports/sos.html ]; then + add_failure "html report generated when --no-report used" + fi + if [ ! $(grep /tmp/sosreport-*TEST* /dev/shm/stdout) ]; then + add_failure "no label set on archive" + fi + count=$(find /tmp/sosreport_test/sos_commands/* -type d | wc -l) + if [[ "$count" -gt 1 ]]; then + add_failure "more than one plugin ran when using -o hardware" + fi + update_failures + fi + update_summary "$cmd" +} + +# test using mask +test_mask () { + cmd="--mask" + run_expecting_success "$cmd" extract + if [ $? -eq 0 ]; then + if [ ! $(grep host0 /tmp/sosreport_test/hostname) ]; then + add_failure "hostname not obfuscated with --mask" + fi + # we don't yet support binary obfuscation, so skip binary matches + if [ "$(grep -rI `hostname` /tmp/sosreport_test/*)" ]; then + add_failure "hostname not obfuscated in all places" + echo "$(grep -rI `hostname` /tmp/sosreport_test/*)" + fi + # only tests first interface + mac_addr=$(cat /sys/class/net/$(ip route show default | awk '/default/ {print $5}')/address) + if [ "$(grep -rI $mac_addr /tmp/sosreport_test/*)" ]; then + add_failure "MAC address not obfuscated in all places" + echo "$(grep -rI $mac_addr /tmp/sosreport_test/*)" + fi + # only tests first interface + ip_addr=$(ip route show default | awk '/default/ {print $3}') + if [ "$(grep -rI $ip_addr /tmp/sosreport_test/*)" ]; then + add_failure "IP address not obfuscated in all places" + echo "$(grep -rI $ip_addr /tmp/sosreport_test/*)" + fi + update_failures + fi + update_summary "$cmd" +} + +# test log-size, env vars, and compression type +test_logsize_env_gzip () { + cmd="--log-size 0 --no-env-vars -z gzip" + run_expecting_success "$cmd" extract + if [ $? -eq 0 ]; then + if [ -f /tmp/sosreport_test/environment ]; then + add_failure "env vars captured when using --no-env-vars" + fi + if [ ! $(grep /tmp/sosreport*.gz /dev/shm/stdout) ]; then + add_failure "archive was not gzip compressed using -z gzip" + fi + update_failures + fi + update_summary "$cmd" +} + +# test plugin enablement, plugopts and at the same time ensure our list option parsing is working +test_enable_opts_postproc () { + cmd="-e opencl -v -k kernel.with-timer,libraries.ldconfigv --no-postproc" + run_expecting_success "$cmd" extract + if [ $? -eq 0 ]; then + if [ ! "$(grep "opencl" /dev/shm/stdout)" ]; then + add_failure "force enabled plugin opencl did not run" + fi + if [ ! -f /tmp/sosreport_test/proc/timer* ]; then + add_failure "/proc/timer* not captured when using -k kernel.with-timer" + fi + if [ ! -f /tmp/sosreport_test/sos_commands/libraries/ldconfig_-v* ]; then + add_failure "ldconfig -v not captured when using -k libraries.ldconfigv" + fi + if [ "$(grep "substituting" /tmp/sosreport_test/sos_logs/sos.log)" ]; then + add_failure "post-processing ran while using --no-post-proc" + fi + + update_failures + update_summary "$cmd" + fi +} + +# test if --build and --threads work properly +test_build_threads () { + cmd="--build -t1 -o host,kernel,filesys,hardware,date,logs" + run_expecting_success "$cmd" + if [ $? -eq 0 ]; then + if [ ! "$(grep "Your sosreport build tree" /dev/shm/stdout)" ]; then + add_failure "did not save the build tree" + fi + if [ $(grep "Finishing plugins" /dev/shm/stdout) ]; then + add_failure "did not limit threads when using --threads 1" + fi + update_failures + update_summary "$cmd" + fi +} + +# If /etc/sos/sos.conf doesn't exist let's just make it +if [ -f /etc/sos/sos.conf ]; then + echo "/etc/sos/sos.conf already exists" +else + echo "Creating /etc/sos.conf" + mkdir /etc/sos + touch /etc/sos/sos.conf +fi + + +# Runs not generating sosreports +run_expecting_success " -l"; update_summary "List plugins" +run_expecting_success " --list-presets"; update_summary "List presets" +run_expecting_success " --list-profiles"; update_summary "List profiles" + +# Runs generating sosreports +# TODO: +# - find a way to test if --since is working +test_build_threads +test_normal_report +test_enable_opts_postproc +test_noreport_label_only +test_logsize_env_gzip +test_mask + +echo -e $summary + +if [ $NUMOFFAILURES -gt 0 ]; then + echo -e "\nTests Failed: $NUMOFFAILURES\nFailures within each test:" + echo -e $FAIL_LIST + exit 1 +else + echo "Everything worked!" + exit 0 +fi + +# vim: set et ts=4 sw=4 : diff --git a/debian/watch b/debian/watch new file mode 100644 index 0000000000..fa4ded0db8 --- /dev/null +++ b/debian/watch @@ -0,0 +1,4 @@ +version=4 + opts="filenamemangle=s%(?:.*?)?v?(\d[\d.]*)\.tar\.gz%sos-$1.tar.gz%" \ + https://github.com/sosreport/sos/tags \ + (?:.*?/)?v?(\d[\d.]*)\.tar\.gz debian uupdate