diff --git a/.devcontainer/Dockerfile b/.devcontainer/Dockerfile new file mode 100644 index 0000000..d0d79a1 --- /dev/null +++ b/.devcontainer/Dockerfile @@ -0,0 +1,10 @@ +FROM fedora:latest + +RUN bash -c "$(curl -fsSL "https://raw.githubusercontent.com/microsoft/vscode-dev-containers/main/script-library/common-redhat.sh")" -- "true" "vscode" "1000" "1000" "true" + +RUN dnf install -y \ + sudo git cargo rust rust-src git-core clippy rustfmt \ + rust-packaging rpmdevtools rpmlint \ + && dnf clean all + +USER vscode diff --git a/.devcontainer/devcontainer.json b/.devcontainer/devcontainer.json new file mode 100644 index 0000000..ec23047 --- /dev/null +++ b/.devcontainer/devcontainer.json @@ -0,0 +1,28 @@ +{ + "name": "greenboot-rs", + "build": { + "dockerfile": "Dockerfile" + }, + "runArgs": [ + "--cap-add=SYS_PTRACE", + "--security-opt", + "seccomp=unconfined" + ], + "settings": { + "rust-analyzer.checkOnSave.command": "clippy" + }, + "extensions": [ + "mutantdino.resourcemonitor", + "matklad.rust-analyzer", + "serayuzgur.crates" + ], + "hostRequirements": { + "memory": "4gb" + }, + "remoteUser": "vscode", + "updateContentCommand": [ + "cargo", + "build" + ], + "waitFor": "onCreateCommand" +} diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index dd37da8..bddaa37 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -1,7 +1,7 @@ on: push: branches: - - main + - greenboot-rs pull_request: name: Continuous integration @@ -22,20 +22,17 @@ jobs: skip: "./docs/Gemfile.lock,./docs/_config.yml,./.github,./.git,./greenboot.spec,./dist" fmt: - name: Rustfmt + name: Cargo fmt runs-on: ubuntu-latest steps: - uses: actions/checkout@v3 - uses: actions-rs/toolchain@v1 with: - profile: minimal toolchain: stable - override: true - - run: rustup component add rustfmt - uses: actions-rs/cargo@v1 with: command: fmt - args: --all -- --check + args: --check --all clippy: name: Clippy @@ -63,11 +60,13 @@ jobs: build_and_test: runs-on: ubuntu-latest - container: fedora:latest + container: + image: fedora:latest + options: --user root steps: - name: Install deps run: | - dnf install -y make gcc git cargo rust git clevis + dnf install -y make gcc git cargo rust git grub2-efi grub2-efi-modules shim - uses: actions/checkout@v3 with: persist-credentials: false @@ -91,51 +90,5 @@ jobs: uses: actions-rs/cargo@v1 with: command: test - # This is primarily to ensure that changes to fdo_data.h are committed, - # which is critical for determining whether any stability changes were made - # during the PR review. - - name: Ensure building did not change any code - run: | - git diff --exit-code - - commitlint: - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v3 - with: - fetch-depth: 0 - - uses: actions/setup-node@v3 - with: - node-version: 'latest' - - name: Install commitlint dependencies - run: npm install commitlint - - uses: wagoid/commitlint-github-action@v5 - env: - NODE_PATH: ${{ github.workspace }}/node_modules - with: - configFile: .github/commitlint.config.js - failOnWarnings: true - - # manpages: - # name: Test man page generation - # runs-on: ubuntu-latest - # container: fedora:latest - # steps: - # - uses: actions/checkout@v3 - # - name: install deps - # run: | - # dnf install -y make python3-docutils - # - name: generate man pages - # run: make man - - # devcontainer_test: - # name: Test Devcontainer Creation - # runs-on: ubuntu-latest - # steps: - # - uses: actions/checkout@v3 - # - name: Install devcontainer CLI - # run: npm install -g @vscode/dev-container-cli - # - name: Build devcontainer - # run: devcontainer build --image-name devcontainer-fdo-rs . - # - name: Test building in devcontainer - # run: docker run --rm -v `pwd`:/code:z --workdir /code --user root devcontainer-fdo-rs cargo build --verbose + args: -- --test-threads=1 + diff --git a/.packit.yaml b/.packit.yaml index a2f8f86..01108d0 100644 --- a/.packit.yaml +++ b/.packit.yaml @@ -19,6 +19,8 @@ srpm_build_deps: # post-upstream-clone: "./tools/rpm_spec_add_provides_bundle.sh" actions: + get-current-version: + - grep -oP '^Version:\s+\K\S+' greenboot.spec create-archive: - "cargo vendor vendor" - bash -c "git archive --prefix=greenboot-${PACKIT_PROJECT_VERSION}/ --format=tar HEAD > greenboot-${PACKIT_PROJECT_VERSION}.tar" diff --git a/Cargo.toml b/Cargo.toml index 37051a1..4ad8304 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -11,7 +11,8 @@ log = "0.4" clap = { version = "4.0", features = ["derive"] } config = "0.13" pretty_env_logger = "0.4" -nix = "0.25.0" +nix = "0.26.2" glob = "0.3.0" serde = "1.0" serde_json = "1.0" +thiserror = "1.0.38" \ No newline at end of file diff --git a/Makefile b/Makefile index 7a358fd..56ce816 100644 --- a/Makefile +++ b/Makefile @@ -42,7 +42,7 @@ install: build .PHONY: check check: - cargo test "--target-dir=${TARGETDIR}" + cargo test "--target-dir=${TARGETDIR}" -- --test-threads=1 .PHONY: srpm srpm: $(RPM_SPECFILE) $(RPM_TARBALL) $(VENDOR_TARBALL) diff --git a/dist/systemd/system/greenboot-trigger.service b/dist/systemd/system/greenboot-trigger.service deleted file mode 100644 index 3dfd0bf..0000000 --- a/dist/systemd/system/greenboot-trigger.service +++ /dev/null @@ -1,21 +0,0 @@ -[Unit] -Description=Greenboot - TODO 2 -DefaultDependencies=no -Conflicts=shutdown.target -Before=shutdown.target - -Wants=local-fs.target -After=local-fs.target - -Before=multi-user.target systemd-update-done.service -ConditionNeedsUpdate=|/etc -ConditionNeedsUpdate=|/var - -[Service] -Type=oneshot -RemainAfterExit=true -ExecStart=/usr/libexec/greenboot/greenboot stamp -Restart=no - -[Install] -WantedBy=multi-user.target diff --git a/dist/systemd/system/greenboot.service b/dist/systemd/system/greenboot.service deleted file mode 100644 index 1d525d6..0000000 --- a/dist/systemd/system/greenboot.service +++ /dev/null @@ -1,13 +0,0 @@ -[Unit] -Description=Greenboot - TODO -After=multi-user.target -Before=boot-complete.target - -[Service] -Type=oneshot -RemainAfterExit=yes -ExecStart=/usr/libexec/greenboot/greenboot check -Restart=no - -[Install] -RequiredBy=boot-complete.target diff --git a/etc/greenboot/greenboot.conf b/etc/greenboot/greenboot.conf new file mode 100644 index 0000000..b989931 --- /dev/null +++ b/etc/greenboot/greenboot.conf @@ -0,0 +1,4 @@ +# Greenboot configuration file + +## Generic +GREENBOOT_MAX_BOOT_ATTEMPTS=3 \ No newline at end of file diff --git a/greenboot.spec b/greenboot.spec index 1424bb2..30efc22 100644 --- a/greenboot.spec +++ b/greenboot.spec @@ -1,17 +1,17 @@ %global debug_package %{nil} %bcond_without check %global with_bundled 1 -%global with_packit 1 +%global with_packit 0 %global __cargo_skip_build 0 %global __cargo_is_lib() false %global forgeurl https://github.com/fedora-iot/greenboot -Version: 0.99.0 +Version: 0.16.0 %forgemeta Name: greenboot -Release: 1%{?dist} +Release: 0%{?dist} Summary: Generic Health Check Framework for systemd License: LGPLv2+ @@ -41,99 +41,122 @@ Requires: rpm-ostree Requires: pam >= 1.4.0 # While not strictly necessary to generate the motd, the main use-case of this package is to display it on SSH login Recommends: openssh -Provides: greenboot-auto-update-fallback -Obsoletes: greenboot-auto-update-fallback <= 0.12.0 -Provides: greenboot-grub2 -Obsoletes: greenboot-grub2 <= 0.12.0 -Provides: greenboot-reboot -Obsoletes: greenboot-reboot <= 0.12.0 -Provides: greenboot-status -Obsoletes: greenboot-status <= 0.12.0 -Provides: greenboot-rpm-ostree-grub2 -Obsoletes: greenboot-rpm-ostree-grub2 <= 0.12.0 -# List of bundled crate in vendor tarball, generated with: +# List of bundled crates in vendor tarball, generated with: # cargo metadata --locked --format-version 1 | CRATE_NAME="greenboot" ./bundled-provides.jq Provides: bundled(crate(ahash)) = 0.7.6 -Provides: bundled(crate(aho-corasick)) = 0.7.19 -Provides: bundled(crate(anyhow)) = 1.0.65 -Provides: bundled(crate(async-trait)) = 0.1.57 +Provides: bundled(crate(aho-corasick)) = 0.7.20 +Provides: bundled(crate(anstream)) = 0.2.6 +Provides: bundled(crate(anstyle)) = 0.3.5 +Provides: bundled(crate(anstyle-parse)) = 0.1.1 +Provides: bundled(crate(anstyle-wincon)) = 0.2.0 +Provides: bundled(crate(anyhow)) = 1.0.70 +Provides: bundled(crate(async-trait)) = 0.1.68 Provides: bundled(crate(atty)) = 0.2.14 Provides: bundled(crate(autocfg)) = 1.1.0 -Provides: bundled(crate(base64)) = 0.13.0 +Provides: bundled(crate(base64)) = 0.13.1 Provides: bundled(crate(bitflags)) = 1.3.2 -Provides: bundled(crate(block-buffer)) = 0.10.3 +Provides: bundled(crate(block-buffer)) = 0.10.4 +Provides: bundled(crate(cc)) = 1.0.79 Provides: bundled(crate(cfg-if)) = 1.0.0 -Provides: bundled(crate(clap)) = 4.0.4 -Provides: bundled(crate(clap_derive)) = 4.0.1 -Provides: bundled(crate(clap_lex)) = 0.3.0 -Provides: bundled(crate(config)) = 0.13.2 -Provides: bundled(crate(cpufeatures)) = 0.2.5 +Provides: bundled(crate(clap)) = 4.2.0 +Provides: bundled(crate(clap_builder)) = 4.2.0 +Provides: bundled(crate(clap_derive)) = 4.2.0 +Provides: bundled(crate(clap_lex)) = 0.4.1 +Provides: bundled(crate(concolor-override)) = 1.0.0 +Provides: bundled(crate(concolor-query)) = 0.3.3 +Provides: bundled(crate(config)) = 0.13.3 +Provides: bundled(crate(cpufeatures)) = 0.2.6 Provides: bundled(crate(crypto-common)) = 0.1.6 -Provides: bundled(crate(digest)) = 0.10.5 +Provides: bundled(crate(digest)) = 0.10.6 Provides: bundled(crate(dlv-list)) = 0.3.0 Provides: bundled(crate(env_logger)) = 0.7.1 -Provides: bundled(crate(generic-array)) = 0.14.6 -Provides: bundled(crate(getrandom)) = 0.2.7 -Provides: bundled(crate(glob)) = 0.3.0 +Provides: bundled(crate(errno)) = 0.3.0 +Provides: bundled(crate(errno-dragonfly)) = 0.1.2 +Provides: bundled(crate(figlet-rs)) = 0.1.5 +Provides: bundled(crate(generic-array)) = 0.14.7 +Provides: bundled(crate(getrandom)) = 0.2.8 +Provides: bundled(crate(glob)) = 0.3.1 Provides: bundled(crate(hashbrown)) = 0.12.3 -Provides: bundled(crate(heck)) = 0.4.0 +Provides: bundled(crate(heck)) = 0.4.1 Provides: bundled(crate(hermit-abi)) = 0.1.19 +Provides: bundled(crate(hermit-abi)) = 0.3.1 Provides: bundled(crate(humantime)) = 1.3.0 -Provides: bundled(crate(itoa)) = 1.0.3 +Provides: bundled(crate(io-lifetimes)) = 1.0.9 +Provides: bundled(crate(is-terminal)) = 0.4.6 +Provides: bundled(crate(itoa)) = 1.0.6 Provides: bundled(crate(json5)) = 0.4.1 Provides: bundled(crate(lazy_static)) = 1.4.0 -Provides: bundled(crate(libc)) = 0.2.133 +Provides: bundled(crate(libc)) = 0.2.140 Provides: bundled(crate(linked-hash-map)) = 0.5.6 +Provides: bundled(crate(linux-raw-sys)) = 0.3.0 Provides: bundled(crate(log)) = 0.4.17 Provides: bundled(crate(memchr)) = 2.5.0 Provides: bundled(crate(memoffset)) = 0.6.5 Provides: bundled(crate(minimal-lexical)) = 0.2.1 -Provides: bundled(crate(nix)) = 0.25.0 -Provides: bundled(crate(nom)) = 7.1.1 -Provides: bundled(crate(once_cell)) = 1.15.0 +Provides: bundled(crate(nix)) = 0.25.1 +Provides: bundled(crate(nom)) = 7.1.3 +Provides: bundled(crate(once_cell)) = 1.17.1 Provides: bundled(crate(ordered-multimap)) = 0.4.3 -Provides: bundled(crate(os_str_bytes)) = 6.3.0 Provides: bundled(crate(pathdiff)) = 0.2.1 -Provides: bundled(crate(pest)) = 2.3.1 -Provides: bundled(crate(pest_derive)) = 2.3.1 -Provides: bundled(crate(pest_generator)) = 2.3.1 -Provides: bundled(crate(pest_meta)) = 2.3.1 +Provides: bundled(crate(pest)) = 2.5.6 +Provides: bundled(crate(pest_derive)) = 2.5.6 +Provides: bundled(crate(pest_generator)) = 2.5.6 +Provides: bundled(crate(pest_meta)) = 2.5.6 Provides: bundled(crate(pin-utils)) = 0.1.0 Provides: bundled(crate(pretty_env_logger)) = 0.4.0 -Provides: bundled(crate(proc-macro-error)) = 1.0.4 -Provides: bundled(crate(proc-macro-error-attr)) = 1.0.4 -Provides: bundled(crate(proc-macro2)) = 1.0.43 +Provides: bundled(crate(proc-macro2)) = 1.0.54 Provides: bundled(crate(quick-error)) = 1.2.3 -Provides: bundled(crate(quote)) = 1.0.21 -Provides: bundled(crate(regex)) = 1.6.0 -Provides: bundled(crate(regex-syntax)) = 0.6.27 +Provides: bundled(crate(quote)) = 1.0.26 +Provides: bundled(crate(regex)) = 1.7.3 +Provides: bundled(crate(regex-syntax)) = 0.6.29 Provides: bundled(crate(ron)) = 0.7.1 Provides: bundled(crate(rust-ini)) = 0.18.0 -Provides: bundled(crate(ryu)) = 1.0.11 -Provides: bundled(crate(serde)) = 1.0.144 -Provides: bundled(crate(serde_derive)) = 1.0.144 -Provides: bundled(crate(serde_json)) = 1.0.85 -Provides: bundled(crate(sha1)) = 0.10.5 +Provides: bundled(crate(rustix)) = 0.37.4 +Provides: bundled(crate(ryu)) = 1.0.13 +Provides: bundled(crate(serde)) = 1.0.159 +Provides: bundled(crate(serde_derive)) = 1.0.159 +Provides: bundled(crate(serde_json)) = 1.0.95 +Provides: bundled(crate(sha2)) = 0.10.6 Provides: bundled(crate(strsim)) = 0.10.0 -Provides: bundled(crate(syn)) = 1.0.100 -Provides: bundled(crate(termcolor)) = 1.1.3 -Provides: bundled(crate(thiserror)) = 1.0.35 -Provides: bundled(crate(thiserror-impl)) = 1.0.35 -Provides: bundled(crate(toml)) = 0.5.9 -Provides: bundled(crate(typenum)) = 1.15.0 +Provides: bundled(crate(syn)) = 1.0.109 +Provides: bundled(crate(syn)) = 2.0.11 +Provides: bundled(crate(termcolor)) = 1.2.0 +Provides: bundled(crate(thiserror)) = 1.0.40 +Provides: bundled(crate(thiserror-impl)) = 1.0.40 +Provides: bundled(crate(toml)) = 0.5.11 +Provides: bundled(crate(typenum)) = 1.16.0 Provides: bundled(crate(ucd-trie)) = 0.1.5 -Provides: bundled(crate(unicode-ident)) = 1.0.4 +Provides: bundled(crate(unicode-ident)) = 1.0.8 +Provides: bundled(crate(utf8parse)) = 0.2.1 Provides: bundled(crate(version_check)) = 0.9.4 Provides: bundled(crate(wasi)) = 0.11.0+wasi_snapshot_preview1 Provides: bundled(crate(winapi)) = 0.3.9 Provides: bundled(crate(winapi-i686-pc-windows-gnu)) = 0.4.0 Provides: bundled(crate(winapi-util)) = 0.1.5 Provides: bundled(crate(winapi-x86_64-pc-windows-gnu)) = 0.4.0 +Provides: bundled(crate(windows-sys)) = 0.45.0 +Provides: bundled(crate(windows-targets)) = 0.42.2 +Provides: bundled(crate(windows_aarch64_gnullvm)) = 0.42.2 +Provides: bundled(crate(windows_aarch64_msvc)) = 0.42.2 +Provides: bundled(crate(windows_i686_gnu)) = 0.42.2 +Provides: bundled(crate(windows_i686_msvc)) = 0.42.2 +Provides: bundled(crate(windows_x86_64_gnu)) = 0.42.2 +Provides: bundled(crate(windows_x86_64_gnullvm)) = 0.42.2 +Provides: bundled(crate(windows_x86_64_msvc)) = 0.42.2 Provides: bundled(crate(yaml-rust)) = 0.4.5 %description %{summary}. +%package default-health-checks +Summary: Series of optional and curated health checks +Requires: %{name} = %{version}-%{release} +Requires: util-linux +Requires: jq + +%description default-health-checks +%{summary}. + %prep %forgeautosetup %if ! 0%{?with_packit} @@ -147,8 +170,15 @@ cat >.cargo/config << EOF [build] rustc = "%{__rustc}" rustdoc = "%{__rustdoc}" +%if 0%{?rhel} && !0%{?eln} rustflags = %{__global_rustflags_toml} - +%else +rustflags = "%{__global_rustflags_toml}" +%endif + +[profile.rpm] +inherits = "release" + [install] root = "%{buildroot}%{_prefix}" @@ -178,10 +208,11 @@ mkdir -p %{buildroot}%{_libexecdir} mkdir -p %{buildroot}%{_libexecdir}/%{name} mv %{buildroot}%{_bindir}/greenboot %{buildroot}%{_libexecdir}/%{name}/%{name} install -Dpm0644 -t %{buildroot}%{_unitdir} \ - dist/systemd/system/*.service + usr/lib/systemd/system/*.service # add config mkdir -p %{buildroot}%{_exec_prefix}/lib/motd.d/ mkdir -p %{buildroot}%{_libexecdir}/%{name} +install -Dpm0644 -t %{buildroot}%{_sysconfdir}/%{name} etc/greenboot/greenboot.conf mkdir -p %{buildroot}%{_sysconfdir}/%{name}/check/required.d mkdir %{buildroot}%{_sysconfdir}/%{name}/check/wanted.d mkdir %{buildroot}%{_sysconfdir}/%{name}/green.d @@ -191,28 +222,30 @@ mkdir %{buildroot}%{_prefix}/lib/%{name}/check/wanted.d mkdir %{buildroot}%{_prefix}/lib/%{name}/green.d mkdir %{buildroot}%{_prefix}/lib/%{name}/red.d mkdir -p %{buildroot}%{_unitdir} -mkdir -p %{buildroot}%{_unitdir}/greenboot-healthcheck.service.d mkdir -p %{buildroot}%{_tmpfilesdir} +install -DpZm 0755 usr/lib/greenboot/check/required.d/* %{buildroot}%{_prefix}/lib/%{name}/check/required.d +install -DpZm 0755 usr/lib/greenboot/check/wanted.d/* %{buildroot}%{_prefix}/lib/%{name}/check/wanted.d %post -%systemd_post greenboot.service -%systemd_post greenboot-trigger.service +%systemd_post greenboot-healthcheck.service +%systemd_post greenboot-rollback.service %preun -%systemd_preun greenboot.service -%systemd_preun greenboot-trigger.service +%systemd_preun greenboot-healthcheck.service +%systemd_preun greenboot-rollback.service %postun -%systemd_postun greenboot.service -%systemd_postun greenboot-trigger.service +%systemd_postun greenboot-healthcheck.service +%systemd_postun greenboot-rollback.service %files %doc README.md %license LICENSE %dir %{_libexecdir}/%{name} %{_libexecdir}/%{name}/%{name} -%{_unitdir}/greenboot.service -%{_unitdir}/greenboot-trigger.service +%{_unitdir}/greenboot-healthcheck.service +%{_unitdir}/greenboot-rollback.service +%{_sysconfdir}/%{name}/greenboot.conf %dir %{_prefix}/lib/%{name} %dir %{_prefix}/lib/%{name}/check %dir %{_prefix}/lib/%{name}/check/required.d @@ -226,6 +259,11 @@ mkdir -p %{buildroot}%{_tmpfilesdir} %dir %{_sysconfdir}/%{name}/green.d %dir %{_sysconfdir}/%{name}/red.d +%files default-health-checks +%{_prefix}/lib/%{name}/check/required.d/01_repository_dns_check.sh +%{_prefix}/lib/%{name}/check/wanted.d/01_update_platforms_check.sh +%{_prefix}/lib/%{name}/check/required.d/02_watchdog.sh + %changelog * Thu Sep 08 2022 Peter Robinson - 0.15.2-1 - The 0.15.2 release diff --git a/src/handler/mod.rs b/src/handler/mod.rs new file mode 100644 index 0000000..53a8ccb --- /dev/null +++ b/src/handler/mod.rs @@ -0,0 +1,109 @@ +/// This module contains most of the low-level commands +/// and grub variable modifications +use anyhow::{bail, Ok, Result}; + +use std::process::Command; +use std::str; + +/// reboots the system if boot_counter is greater than 0 or can be forced too +pub fn handle_reboot(force: bool) -> Result<()> { + if !force { + let boot_counter = get_boot_counter()?; + if boot_counter <= Some(0) { + bail!("countdown ended, check greenboot-rollback status") + }; + } + log::info!("restarting the system"); + Command::new("systemctl").arg("reboot").status()?; + Ok(()) +} + +/// rollback to previous ostree deployment if boot counter is less than 0 +pub fn handle_rollback() -> Result<()> { + let boot_counter = get_boot_counter()?; + if boot_counter <= Some(0) { + log::info!("Greenboot will now attempt to rollback"); + Command::new("rpm-ostree").arg("rollback").status()?; + return Ok(()); + } + bail!("Rollback not initiated"); +} + +/// sets grub variable boot_counter if not set +pub fn set_boot_counter(reboot_count: u16) -> Result<()> { + let current_counter= get_boot_counter()?; + match current_counter{ + Some(counter) => { + log::info!("boot_counter={counter}"); + bail!("counter already set"); + } + _ => { + //will still try to set boot_counter to override cases like boot_counter= + log::info!("setting boot counter"); + set_grub_var("boot_counter", reboot_count)?; + } + } + Ok(()) +} + +/// resets grub variable boot_counter +pub fn unset_boot_counter() -> Result<()> { + Command::new("grub2-editenv") + .arg("-") + .arg("unset") + .arg("boot_counter") + .status()?; + Ok(()) +} + +/// sets grub variable boot_success +pub fn set_boot_status(success: bool) -> Result<()> { + if success { + set_grub_var("boot_success", 1)?; + unset_boot_counter()?; + return Ok(()); + } + set_grub_var("boot_success", 0) +} + +/// writes greenboot status to motd.d/boot-status +pub fn handle_motd(state: &str) -> Result<()> { + std::fs::write( + "/etc/motd.d/boot-status", + format!("Greenboot {state}.").as_bytes(), + )?; + Ok(()) +} + +/// fetches boot_counter value, none if not set +pub fn get_boot_counter() -> Result> { + let grub_vars = Command::new("grub2-editenv") + .arg("-") + .arg("list") + .output()?; + let grub_vars = str::from_utf8(&grub_vars.stdout[..])?; + for var in grub_vars.lines() { + let (k, v) = if let Some(kv) = var.split_once('=') { + kv + } else { + continue; + }; + if k != "boot_counter" { + continue; + } + + let counter_value = v.parse::()?; + return Ok(Some(counter_value)); + } + Ok(None) +} + +/// helper function to set any grub variable +fn set_grub_var(key: &str, val: u16) -> Result<()> { + Command::new("grub2-editenv") + .arg("-") + .arg("set") + .arg(format!("{key}={val}")) + .status()?; + Ok(()) +} diff --git a/src/main.rs b/src/main.rs index a55df5b..6eeb05c 100644 --- a/src/main.rs +++ b/src/main.rs @@ -1,29 +1,79 @@ -use std::hash::Hash; -use std::io::ErrorKind; -use std::iter::FromIterator; -use std::{ - collections::HashSet, - fs::{self, File}, - process::Command, -}; - -use anyhow::{bail, Error, Result}; +mod handler; +use anyhow::{bail, Result}; use clap::{Parser, Subcommand, ValueEnum}; +use config::{Config, File, FileFormat}; use glob::glob; -use serde::{Deserialize, Serialize}; +use handler::*; +use serde::Deserialize; +use std::error::Error; +use std::path::Path; +use std::process::Command; +use std::str; + +/// dir that greenboot looks for the health check and other scripts +static GREENBOOT_INSTALL_PATHS: [&str; 2] = ["/usr/lib/greenboot", "/etc/greenboot"]; + +/// greenboot config path +static GREENBOOT_CONFIG_FILE: &str = "/etc/greenboot/greenboot.conf"; #[derive(Parser)] #[clap(author, version, about, long_about = None)] #[clap(propagate_version = true)] + +/// cli parameters for greenboot struct Cli { #[clap(value_enum, short, long, default_value_t = LogLevel::Info)] log_level: LogLevel, - #[clap(subcommand)] command: Commands, } +#[derive(Debug, Deserialize)] +/// config params for greenboot +struct GreenbootConfig { + max_reboot: u16, +} + +impl GreenbootConfig { + /// sets the default parameter for greenboot config + fn set_default() -> Self { + Self { max_reboot: 3 } + } + /// gets the config from the config file + fn get_config() -> Self { + let mut config = Self::set_default(); + let parsed = Config::builder() + .add_source(File::new(GREENBOOT_CONFIG_FILE, FileFormat::Ini)) + .build(); + match parsed { + Ok(c) => { + config.max_reboot = match c.get_int("GREENBOOT_MAX_BOOT_ATTEMPTS") { + Ok(c) => c.try_into().unwrap_or_else(|e| { + log::warn!( + "{e}, config error, using default value: {}", + config.max_reboot + ); + config.max_reboot + }), + Err(e) => { + log::warn!( + "{e}, config error, using default value: {}", + config.max_reboot + ); + config.max_reboot + } + } + } + Err(e) => log::warn!( + "{e}, config error, using default value: {}", + config.max_reboot + ), + } + config + } +} #[derive(Copy, Clone, PartialEq, Eq, PartialOrd, Ord, ValueEnum)] +/// log level for journald logging enum LogLevel { Trace, Debug, @@ -47,125 +97,289 @@ impl LogLevel { } #[derive(Subcommand)] +/// params that greenboot accepts +/// +/// greenboot health-check -> runs the custom health checks +/// +/// greenboot rollback -> if boot_counter satisfies it trigger rollback enum Commands { - Check, - Stamp, -} - -#[derive(Serialize, Deserialize, PartialEq, Eq, Hash)] -struct ServiceStatus { - unit: String, + HealthCheck, + Rollback, } -fn check() -> Result<(), Error> { - match File::open("/etc/greenboot/upgrade.stamp") { - Ok(_) => { - log::info!("stamp on disk, removing and running greenboot"); - std::fs::remove_file("/etc/greenboot/upgrade.stamp")? +/// runs all the scripts in required.d and wanted.d +fn run_diagnostics() -> Result<()> { + let mut required_script_failure: bool = false; + let mut path_exists: bool = false; + for path in GREENBOOT_INSTALL_PATHS { + let greenboot_required_path = format!("{path}/check/required.d/"); + if !Path::new(&greenboot_required_path).is_dir() { + log::warn!("skipping test as {greenboot_required_path} is not a dir"); + continue; } - Err(e) => match e.kind() { - ErrorKind::NotFound => return Ok(()), - _ => { - bail!("unknown error when opening stamp file: {:?}", e); - } - }, - } - let mut failure = false; - for path in [ - "/usr/lib/greenboot/check/required.d/*.sh", - "/etc/greenboot/check/required.d/*.sh", - ] { - for entry in glob(path)?.flatten() { - log::info!("running required check {}", entry.to_string_lossy()); - let output = Command::new("bash").arg("-C").arg(entry).output()?; - if !output.status.success() { - // combine and print stderr/stdout - log::warn!("required script failed..."); - failure = true; + path_exists = true; + let errors = run_scripts("required", &greenboot_required_path); + if !errors.is_empty() { + log::error!("required script error:"); + errors.iter().for_each(|e| log::error!("{e}")); + if !required_script_failure { + required_script_failure = true; } } } - for path in [ - "/usr/lib/greenboot/check/wanted.d/*.sh", - "/etc/greenboot/check/wanted.d/*.sh", - ] { - for entry in glob(path)?.flatten() { - log::info!("running required check {}", entry.to_string_lossy()); - let output = Command::new("bash").arg("-C").arg(entry).output()?; - if !output.status.success() { - // combine and print stderr/stdout - log::warn!("wanted script failed..."); - } + if !path_exists { + bail!("cannot find any required.d folder"); + } + for path in GREENBOOT_INSTALL_PATHS { + let greenboot_wanted_path = format!("{path}/check/wanted.d/"); + let errors = run_scripts("wanted", &greenboot_wanted_path); + if !errors.is_empty() { + log::warn!("wanted script runner error:"); + errors.iter().for_each(|e| log::error!("{e}")); } } - // if a command with restart option in systemd fails to start we don't get it as "failed" - // reversing the check makes sure that if by the time After=multi-user the service isn't running then it's failing at least - let output = Command::new("systemctl") - .arg("list-units") - .arg("--state") - .arg("active") - .arg("--no-page") - .arg("--output") - .arg("json") - .output()?; - let services: Vec = serde_json::from_str(&String::from_utf8(output.stdout)?)?; - let ss: Vec = services.iter().map(|x| x.unit.clone()).collect(); - let active_units: HashSet = HashSet::from_iter(ss); - for service in ["sshd.service", "NetworkManager.service"] { - if !active_units.contains(service) { - log::warn!("service {} failed, see journal", service); - failure = true; + + if required_script_failure { + bail!("health-check failed!"); + } + Ok(()) +} + +/// runs all the scripts in red.d when health-check fails +fn run_red() -> Vec> { + let mut errors = Vec::new(); + for path in GREENBOOT_INSTALL_PATHS { + let red_path = format!("{path}/red.d/"); + let e = run_scripts("red", &red_path); + if !e.is_empty() { + errors.extend(e); } } - if failure { - for path in ["/etc/greenboot/red.d/*.sh"] { - for entry in glob(path)?.flatten() { - log::info!("running red check {}", entry.to_string_lossy()); - let output = Command::new("bash").arg("-C").arg(entry).output()?; - if !output.status.success() { - // combine and print stderr/stdout - log::warn!("red script failed..."); - } + errors +} + +/// runs all the scripts green.d when health-check passes +fn run_green() -> Vec> { + let mut errors = Vec::new(); + for path in GREENBOOT_INSTALL_PATHS { + let green_path = format!("{path}/green.d/"); + let e = run_scripts("green", &green_path); + if !e.is_empty() { + errors.extend(e); + } + } + errors +} + +/// triggers the diagnostics followed by the action on the outcome +/// this also handles setting the grub variables and system restart +fn health_check() -> Result<()> { + let config = GreenbootConfig::get_config(); + log::debug!("{config:?}"); + handle_motd("healthcheck is in progress")?; + match run_diagnostics() { + Ok(()) => { + log::info!("greenboot health-check passed."); + let errors = run_green(); + if !errors.is_empty() { + log::error!("There is a problem with green script runner"); + errors.iter().for_each(|e| log::error!("{e}")); } + handle_motd("healthcheck passed - status is GREEN") + .unwrap_or_else(|e| log::error!("cannot set motd: {}", e.to_string())); + set_boot_status(true)?; + Ok(()) } - log::warn!("SYSTEM is UNHEALTHY. Rolling back and rebooting..."); - Command::new("rpm-ostree").arg("rollback").status()?; - reboot()?; - return Ok(()); - } - for path in ["/etc/greenboot/green.d/*.sh"] { - for entry in glob(path)?.flatten() { - log::info!("running green check {}", entry.to_string_lossy()); - let output = Command::new("bash").arg("-C").arg(entry).output()?; - if !output.status.success() { - // combine and print stderr/stdout - log::warn!("green script failed..."); + Err(e) => { + log::error!("Greenboot error: {e}"); + handle_motd("healthcheck failed - status is RED") + .unwrap_or_else(|e| log::error!("cannot set motd: {}", e.to_string())); + let errors = run_red(); + if !errors.is_empty() { + log::error!("There is a problem with red script runner"); + errors.iter().for_each(|e| log::error!("{e}")); } + + set_boot_status(false)?; + set_boot_counter(config.max_reboot) + .unwrap_or_else(|e| log::error!("cannot set boot_counter: {}", e.to_string())); + handle_reboot(false) + .unwrap_or_else(|e| log::error!("cannot reboot: {}", e.to_string())); + Err(e) } } - Ok(()) } -fn reboot() -> Result<(), Error> { - Command::new("systemctl").arg("reboot").spawn()?; - Ok(()) +/// initiates rollback if boot_counter and satisfies +fn trigger_rollback() -> Result<()> { + match handle_rollback() { + Ok(()) => { + log::info!("Rollback successful"); + unset_boot_counter()?; + handle_reboot(true) + } + Err(e) => { + bail!("{e}, Rollback is not initiated"); + } + } } -fn stamp() -> Result<(), Error> { - fs::create_dir_all("/etc/greenboot/")?; - File::create("/etc/greenboot/upgrade.stamp")?; - Ok(()) +/// takes in a path and runs all the .sh files within the path +/// returns false if any script fails +fn run_scripts(name: &str, path: &str) -> Vec> { + let mut errors = Vec::new(); + let scripts = format!("{path}*.sh"); + match glob(&scripts) { + Ok(s) => { + for entry in s.flatten() { + log::info!("running {name} check {}", entry.to_string_lossy()); + let output = Command::new("bash").arg("-C").arg(entry.clone()).output(); + match output { + Ok(o) => { + if !o.status.success() { + errors.push(Box::new(std::io::Error::new( + std::io::ErrorKind::Other, + format!( + "{name} script {} failed! \n{} \n{}", + entry.to_string_lossy(), + String::from_utf8_lossy(&o.stdout), + String::from_utf8_lossy(&o.stderr) + ), + )) as Box); + } else { + log::info!("{name} script {} success!", entry.to_string_lossy()); + } + } + Err(e) => { + errors.push(Box::new(e) as Box); + } + } + } + } + Err(e) => errors.push(Box::new(e) as Box), + } + errors } fn main() -> Result<()> { let cli = Cli::parse(); - pretty_env_logger::formatted_builder() .filter_level(cli.log_level.to_log()) .init(); match cli.command { - Commands::Check => check(), - Commands::Stamp => stamp(), + Commands::HealthCheck => health_check(), + Commands::Rollback => trigger_rollback(), + } +} + +#[cfg(test)] +mod tests { + use std::fs; + + use anyhow::Context; + + use super::*; + + /// validate when the required folder is not found + #[test] + fn missing_required_folder() { + assert_eq!( + run_diagnostics().unwrap_err().to_string(), + String::from("cannot find any required.d folder") + ); + } + + #[test] + fn test_passed_diagnostics() { + setup_folder_structure(true) + .context("Test setup failed") + .unwrap(); + let state = run_diagnostics(); + assert!(state.is_ok()); + tear_down().context("Test teardown failed").unwrap(); + } + + #[test] + fn test_failed_diagnostics() { + setup_folder_structure(false) + .context("Test setup failed") + .unwrap(); + let failed_msg = run_diagnostics().unwrap_err().to_string(); + assert_eq!(failed_msg, String::from("health-check failed!")); + tear_down().context("Test teardown failed").unwrap(); + } + + #[test] + fn test_boot_counter_set() { + unset_boot_counter().ok(); + set_boot_counter(10).ok(); + assert_eq!(get_boot_counter().unwrap(), Some(10)); + unset_boot_counter().ok(); + } + + #[test] + fn test_boot_counter_re_set() { + unset_boot_counter().ok(); + set_boot_counter(10).ok(); + set_boot_counter(20).ok(); + assert_eq!(get_boot_counter().unwrap(), Some(10)); + unset_boot_counter().ok(); + } + + #[test] + fn test_boot_counter_having_invalid_value() { + unset_boot_counter().ok(); + let _ = Command::new("grub2-editenv") + .arg("-") + .arg("set") + .arg("boot_counter=foo") + .spawn() + .context("Cannot create grub variable boot_counter"); + set_boot_counter(13).ok(); + assert_eq!(get_boot_counter().unwrap(), Some(13)); + unset_boot_counter().ok(); + } + + fn setup_folder_structure(passing: bool) -> Result<()> { + let required_path = format!("{}/check/required.d", GREENBOOT_INSTALL_PATHS[1]); + let wanted_path = format!("{}/check/wanted.d", GREENBOOT_INSTALL_PATHS[1]); + let passing_test_scripts = "testing_assets/passing_script.sh"; + let failing_test_scripts = "testing_assets/failing_script.sh"; + + fs::create_dir_all(&required_path).expect("cannot create folder"); + fs::create_dir_all(&wanted_path).expect("cannot create folder"); + let _a = fs::copy( + passing_test_scripts, + format!("{}/passing_script.sh", &required_path), + ) + .context("unable to copy test assets"); + + let _a = fs::copy( + passing_test_scripts, + format!("{}/passing_script.sh", &wanted_path), + ) + .context("unable to copy test assets"); + + let _a = fs::copy( + failing_test_scripts, + format!("{}/failing_script.sh", &wanted_path), + ) + .context("unable to copy test assets"); + + if !passing { + let _a = fs::copy( + failing_test_scripts, + format!("{}/failing_script.sh", &required_path), + ) + .context("unable to copy test assets"); + return Ok(()); + } + Ok(()) + } + + fn tear_down() -> Result<()> { + fs::remove_dir_all(GREENBOOT_INSTALL_PATHS[1]).expect("Unable to delete folder"); + Ok(()) } } diff --git a/testing_assets/failing_script.sh b/testing_assets/failing_script.sh new file mode 100644 index 0000000..fc39ca2 --- /dev/null +++ b/testing_assets/failing_script.sh @@ -0,0 +1,6 @@ +#!/bin/bash +set -euo pipefail + +echo "This is a failing script" + +exit 1 diff --git a/testing_assets/passing_script.sh b/testing_assets/passing_script.sh new file mode 100644 index 0000000..e8022bc --- /dev/null +++ b/testing_assets/passing_script.sh @@ -0,0 +1,4 @@ +#!/bin/bash +set -euo pipefail + +echo "This is a passing script" diff --git a/usr/lib/greenboot/check/required.d/01_repository_dns_check.sh b/usr/lib/greenboot/check/required.d/01_repository_dns_check.sh new file mode 100644 index 0000000..0ff60f8 --- /dev/null +++ b/usr/lib/greenboot/check/required.d/01_repository_dns_check.sh @@ -0,0 +1,56 @@ +#!/bin/bash +set -e + +REPOS_DIRECTORY=/etc/ostree/remotes.d +DOMAINS_WITH_PROBLEMS=() + +get_domain_names_from_platform_urls() { + DOMAIN_NAMES=$(grep -P -ho 'http[s]?\:\/\/[a-zA-Z0-9./-]+' $REPOS_DIRECTORY/* \ + | grep -v -P '.*[0-9]{1,3}\.[0-9]{1,3}\.[0-9]{1,3}\.[0-9]{1,3}' \ + | awk -F:// '{print $2}' \ + | awk -F/ 'BEGIN{OFS="\n"}{print $1}' \ + | sort | uniq) + if [[ -z $DOMAIN_NAMES ]]; then + echo "No domain names have been found" + fi +} + +get_dns_resolution_from_domain_names() { + # Check if each domain name resolves into at least 1 IP + # If it doesn't, add it to DOMAINS_WITH_PROBLEMS + for line in $DOMAIN_NAMES; do + NUMBER_OF_IPS_PER_DOMAIN=$(getent hosts "$line" | wc -l) + if [[ $NUMBER_OF_IPS_PER_DOMAIN -eq 0 ]]; then + DOMAINS_WITH_PROBLEMS+=( "$line" ) + fi + done +} + +assert_dns_resolution_result() { + # If the number of domains with problems is 0, everything's good + # If it's not 0, we exit with errors and print the domains + if [[ ${#DOMAINS_WITH_PROBLEMS[@]} -eq 0 ]]; then + echo "All domains have resolved correctly" + exit 0 + else + echo "The following repository domains haven't responded properly to DNS queries:" + echo "${DOMAINS_WITH_PROBLEMS[*]}" + exit 1 + fi +} + +if [[ ! -d $REPOS_DIRECTORY ]]; then + echo "${REPOS_DIRECTORY} doesn't exist" + exit 1 +fi + +if [ -z "$(ls -A $REPOS_DIRECTORY)" ]; then + echo "${REPOS_DIRECTORY} is empty, skipping check" + exit 0 +fi + +get_domain_names_from_platform_urls +if [[ -n $DOMAIN_NAMES ]]; then + get_dns_resolution_from_domain_names + assert_dns_resolution_result +fi diff --git a/usr/lib/greenboot/check/required.d/02_watchdog.sh b/usr/lib/greenboot/check/required.d/02_watchdog.sh new file mode 100644 index 0000000..482c6d1 --- /dev/null +++ b/usr/lib/greenboot/check/required.d/02_watchdog.sh @@ -0,0 +1,70 @@ +#!/bin/bash +set -eo pipefail + +source_configuration_file() { + GREENBOOT_CONFIGURATION_FILE=/etc/greenboot/greenboot.conf + if test -f "$GREENBOOT_CONFIGURATION_FILE"; then + # shellcheck source=etc/greenboot/greenboot.conf + source $GREENBOOT_CONFIGURATION_FILE + fi +} + +set_grace_period() { + DEFAULT_GRACE_PERIOD=24 # default to 24 hours + + if [ -n "$GREENBOOT_WATCHDOG_GRACE_PERIOD" ]; then + GRACE_PERIOD=$GREENBOOT_WATCHDOG_GRACE_PERIOD + else + GRACE_PERIOD=$DEFAULT_GRACE_PERIOD + fi +} + +check_if_there_is_a_watchdog() { + if wdctl 2>/dev/null ; then + return 0 + else + return 1 + fi +} + +check_if_current_boot_is_wd_triggered() { + if check_if_there_is_a_watchdog ; then + WDCTL_OUTPUT=$(wdctl --flags-only --noheadings | grep -c '1$' || true) + if [ "$WDCTL_OUTPUT" -gt 0 ]; then + # This means the boot was watchdog triggered + # TO-DO: maybe do a rollback here? + echo "Watchdog triggered after recent update" + exit 1 + fi + else + # There's no watchdog, so nothing to be done here + exit 0 + fi +} + +# This is in order to test check_if_current_boot_is_wd_triggered +# function within a container +if [ "${1}" != "--source-only" ]; then + if ! check_if_there_is_a_watchdog ; then + echo "No watchdog on the system, skipping check" + exit 0 + fi + + source_configuration_file + if [ "${GREENBOOT_WATCHDOG_CHECK_ENABLED,,}" != "true" ]; then + echo "Watchdog check is disabled" + exit 0 + fi + + set_grace_period + + SECONDS_IN_AN_HOUR=$((60 * 60)) + LAST_DEPLOYMENT_TIMESTAMP=$(rpm-ostree status --json | jq .deployments[0].timestamp) + + HOURS_SINCE_LAST_UPDATE=$((($(date +%s) - "$LAST_DEPLOYMENT_TIMESTAMP") / SECONDS_IN_AN_HOUR)) + if [ "$HOURS_SINCE_LAST_UPDATE" -lt "$GRACE_PERIOD" ]; then + check_if_current_boot_is_wd_triggered + else + exit 0 + fi +fi diff --git a/usr/lib/greenboot/check/wanted.d/01_update_platforms_check.sh b/usr/lib/greenboot/check/wanted.d/01_update_platforms_check.sh new file mode 100644 index 0000000..35ea691 --- /dev/null +++ b/usr/lib/greenboot/check/wanted.d/01_update_platforms_check.sh @@ -0,0 +1,38 @@ +#!/bin/bash +set -e + +REPOS_DIRECTORY=/etc/ostree/remotes.d +URLS_WITH_PROBLEMS=() + +get_update_platform_urls() { + mapfile -t UPDATE_PLATFORM_URLS < <(grep -P -ho 'http[s]?.*' "${REPOS_DIRECTORY}"/*) + if [[ ${#UPDATE_PLATFORM_URLS[@]} -eq 0 ]]; then + echo "No update platforms found, this can be a mistake" + exit 1 + fi +} + +assert_update_platforms_are_responding() { + for UPDATE_PLATFORM_URL in "${UPDATE_PLATFORM_URLS[@]}"; do + HTTP_STATUS=$(curl -o /dev/null -Isw '%{http_code}\n' "$UPDATE_PLATFORM_URL" || echo "Unreachable") + if ! [[ $HTTP_STATUS == 2* ]] && ! [[ $HTTP_STATUS == 3* ]]; then + URLS_WITH_PROBLEMS+=( "$UPDATE_PLATFORM_URL" ) + fi + done + if [[ ${#URLS_WITH_PROBLEMS[@]} -eq 0 ]]; then + echo "We can connect to all update platforms" + exit 0 + else + echo "There are problems connecting with the following URLs:" + echo "${URLS_WITH_PROBLEMS[*]}" + exit 1 + fi +} + +if [[ ! -d $REPOS_DIRECTORY ]]; then + echo "${REPOS_DIRECTORY} doesn't exist" + exit 1 +fi + +get_update_platform_urls +assert_update_platforms_are_responding diff --git a/usr/lib/systemd/system/greenboot-healthcheck.service b/usr/lib/systemd/system/greenboot-healthcheck.service new file mode 100644 index 0000000..9d0c5d8 --- /dev/null +++ b/usr/lib/systemd/system/greenboot-healthcheck.service @@ -0,0 +1,17 @@ +[Unit] +Description=greenboot Health Checks Runner +DefaultDependencies=no +Before=boot-complete.target systemd-update-done.service +OnFailureJobMode=fail +RequiresMountsFor=/boot +RequiresMountsFor=/etc + +[Service] +Type=oneshot +RemainAfterExit=yes +ExecStart=/usr/libexec/greenboot/greenboot health-check +Restart=no + +[Install] +RequiredBy=boot-complete.target +WantedBy=multi-user.target diff --git a/usr/lib/systemd/system/greenboot-rollback.service b/usr/lib/systemd/system/greenboot-rollback.service new file mode 100644 index 0000000..597c3ec --- /dev/null +++ b/usr/lib/systemd/system/greenboot-rollback.service @@ -0,0 +1,16 @@ + +[Unit] +Description=Greenboot rollback +Requires=dbus.service +RequiresMountsFor=/boot +After=systemd-update-done.service +BindsTo=systemd-update-done.service + +[Service] +Type=oneshot +RemainAfterExit=true +ExecStart=/usr/libexec/greenboot/greenboot rollback +Restart=no + +[Install] +WantedBy=multi-user.target \ No newline at end of file