refactor system boot sequence

The current systemd boot sequence is error prone on isolated scenarios like sending invalid configurations through user data. These scenarios could cause inconsistent states at the end of the boot process, which affect the mechanism to determine if a boot was successful after applying new configurations or updates. It is also difficult to implement features that require service initialization order. In order to fix the problems presented above, this commit defines a new boot sequence, with three main systemd targets: preconfigured, configured and multi-user. Preconfigured This target is used to start the boot process. Failures in any of the required service units will cause the target to fail, stopping the boot process. The boot is marked as successful during the execution of this target only if the migrator "oneshot" service exists successfully. Services initialized during this phase include: * migrator * mark-successful-boot * send-boot-success * storewolf * apiserver * early-boot-config * sundog * settings-applier Once the target is reached (completed), the "activate-configured.service" unit will set the configured target as the default target and start it. Configured This target should be used to setup additional configurations in the host before services like kubernetes/ecs start. Services initialized during this phase include: * chronyd * host-containerd Once the target is reached (completed) the "activate-multi-user.service" unit will set the multi-user target as the default target, and start it. Multi-user This is the final target enabled in the boot sequence. Services initialized during this phase include: * host-containers@* * docker * ecs * kubernetes
foersleo · Apr 2, 2021 · 0e74737 · 0e74737
1 parent e04cdf5
commit 0e74737
Show file tree

Hide file tree

Showing 29 changed files with 191 additions and 55 deletions.
diff --git a/packages/acpid/acpid.service b/packages/acpid/acpid.service
@@ -6,4 +6,4 @@ Type=forking
 ExecStart=/usr/sbin/acpid -c /usr/lib/acpid/events
 
 [Install]
-WantedBy=multi-user.target
+WantedBy=preconfigured.target
diff --git a/packages/chrony/chronyd.service b/packages/chrony/chronyd.service
@@ -1,12 +1,12 @@
 [Unit]
 Description=A versatile implementation of the Network Time Protocol
 Documentation=https://chrony.tuxfamily.org
-After=network-online.target configured.target
-Requires=network-online.target configured.target
+After=network-online.target preconfigured.target
+Wants=network-online.target preconfigured.target
 
 [Service]
 Type=simple
 ExecStart=/usr/sbin/chronyd -d -F -1
 
 [Install]
-WantedBy=multi-user.target
+WantedBy=configured.target
diff --git a/packages/dbus-broker/dbus-broker.service b/packages/dbus-broker/dbus-broker.service
@@ -19,4 +19,4 @@ ExecReload=/usr/bin/busctl call org.freedesktop.DBus /org/freedesktop/DBus org.f
 
 [Install]
 Alias=dbus.service
-WantedBy=multi-user.target
+WantedBy=preconfigured.target
diff --git a/packages/ecs-agent/ecs.service b/packages/ecs-agent/ecs.service
@@ -1,7 +1,7 @@
 [Unit]
 Description=Amazon Elastic Container Service - container agent
 Documentation=https://aws.amazon.com/documentation/ecs/
-Requires=docker.service configured.target
+Requires=docker.service
 After=docker.service configured.target
 Wants=network-online.target configured.target
 

diff --git a/packages/host-ctr/host-containerd.service b/packages/host-ctr/host-containerd.service
@@ -1,8 +1,8 @@
 [Unit]
 Description=containerd runtime for host containers
 Documentation=https://containerd.io
-After=network-online.target configured.target
-Wants=network-online.target configured.target
+After=network-online.target preconfigured.target
+Wants=network-online.target preconfigured.target
 
 [Service]
 EnvironmentFile=/etc/network/proxy.env
@@ -19,4 +19,4 @@ LimitNOFILE=infinity
 TasksMax=infinity
 
 [Install]
-WantedBy=multi-user.target
+WantedBy=configured.target
diff --git a/packages/kubernetes-1.15/kubelet.service b/packages/kubernetes-1.15/kubelet.service
@@ -29,4 +29,3 @@ MemoryAccounting=true
 
 [Install]
 WantedBy=multi-user.target
-RequiredBy=mark-successful-boot.service
diff --git a/packages/kubernetes-1.16/kubelet.service b/packages/kubernetes-1.16/kubelet.service
@@ -29,4 +29,3 @@ MemoryAccounting=true
 
 [Install]
 WantedBy=multi-user.target
-RequiredBy=mark-successful-boot.service
diff --git a/packages/kubernetes-1.17/kubelet.service b/packages/kubernetes-1.17/kubelet.service
@@ -29,4 +29,3 @@ MemoryAccounting=true
 
 [Install]
 WantedBy=multi-user.target
-RequiredBy=mark-successful-boot.service
diff --git a/packages/kubernetes-1.18/kubelet.service b/packages/kubernetes-1.18/kubelet.service
@@ -29,4 +29,3 @@ MemoryAccounting=true
 
 [Install]
 WantedBy=multi-user.target
-RequiredBy=mark-successful-boot.service
diff --git a/packages/kubernetes-1.19/kubelet.service b/packages/kubernetes-1.19/kubelet.service
@@ -29,4 +29,3 @@ MemoryAccounting=true
 
 [Install]
 WantedBy=multi-user.target
-RequiredBy=mark-successful-boot.service
diff --git a/packages/libaudit/audit-rules.service b/packages/libaudit/audit-rules.service
@@ -12,4 +12,4 @@ ExecStart=/sbin/auditctl -R /usr/share/audit/audit.rules
 ExecStop=-/sbin/auditctl -D
 
 [Install]
-WantedBy=multi-user.target
+WantedBy=preconfigured.target
diff --git a/packages/os/apiserver.service b/packages/os/apiserver.service
@@ -9,4 +9,4 @@ ExecStart=/usr/bin/apiserver --datastore-path /var/lib/bottlerocket/datastore/cu
 StandardError=journal+console
 
 [Install]
-WantedBy=multi-user.target
+WantedBy=preconfigured.target
diff --git a/packages/os/early-boot-config.service b/packages/os/early-boot-config.service
@@ -1,11 +1,18 @@
 [Unit]
 Description=Bottlerocket userdata configuration system
 # Need network online to talk to IMDS.
-After=network-online.target apiserver.service
-Requires=network-online.target apiserver.service
+After=network-online.target apiserver.service storewolf.service
+# Don't restart the unit if the network goes offline or apiserver restarts
+Wants=apiserver.service network-online.target
+# Don't start the unit if storewolf.service fails
+Requires=storewolf.service
 # We only want to run once, at first boot.  This file is created by early-boot-config
 # after a successful run.
 ConditionPathExists=!/var/lib/bottlerocket/early-boot-config.ran
+# Block manual interactions with this service, since it could leave the system in an
+# unexpected state
+RefuseManualStart=true
+RefuseManualStop=true
 
 [Service]
 Type=oneshot
@@ -14,4 +21,4 @@ RemainAfterExit=true
 StandardError=journal+console
 
 [Install]
-WantedBy=multi-user.target
+RequiredBy=preconfigured.target
diff --git a/packages/os/[email protected] b/packages/os/[email protected]
@@ -7,8 +7,6 @@ Wants=host-containerd.service
 Type=simple
 EnvironmentFile=/etc/host-containers/%i.env
 Environment=LOCAL_DIR=/local
-# Create directories for container persistent storage
-ExecStartPre=/usr/bin/mkdir -m 1777 -p ${LOCAL_DIR}/host-containers/%i
 ExecStart=/usr/bin/host-ctr run \
     --container-id='%i' \
     --source='${CTR_SOURCE}' \

diff --git a/packages/os/mark-successful-boot.service b/packages/os/mark-successful-boot.service
@@ -1,15 +1,18 @@
 [Unit]
 Description=Call signpost to mark the boot as successful after all required targets are met.
-After=multi-user.target
-# Each service that must start correctly in order for a boot to be successful should be of type "notify"
-# and include "RequiredBy=mark-successful-boot.service" in its [Install] section.
+# This unit is in charge of updating the partitions on successful boots. Use other service
+# units instead of adding more `ExecStart*` lines to prevent indirect dependencies on
+# other units not listed in the `RequiredBy` section.
+Requires=migrator.service
+# Block manual interactions with this service, manually running it could leave the system in an
+# unexpected state
+RefuseManualStart=true
+RefuseManualStop=true
 
 [Service]
-EnvironmentFile=/etc/network/proxy.env
 Type=oneshot
 RemainAfterExit=true
 ExecStart=/bin/signpost mark-successful-boot
-ExecStartPost=-/usr/bin/metricdog send-boot-success
 
 [Install]
-WantedBy=multi-user.target
+RequiredBy=preconfigured.target
diff --git a/packages/os/migrator.service b/packages/os/migrator.service
@@ -1,11 +1,18 @@
 [Unit]
 Description=Bottlerocket data store migrator
+RefuseManualStart=true
+RefuseManualStop=true
 
 [Service]
 Type=oneshot
-ExecStart=/usr/bin/migrator --datastore-path /var/lib/bottlerocket/datastore/current --migration-directory /var/lib/bottlerocket-migrations --root-path /usr/share/updog/root.json --metadata-directory /var/cache/bottlerocket-metadata --migrate-to-version-from-os-release
+ExecStart=/usr/bin/migrator \
+  --datastore-path /var/lib/bottlerocket/datastore/current \
+  --migration-directory /var/lib/bottlerocket-migrations \
+  --root-path /usr/share/updog/root.json \
+  --metadata-directory /var/cache/bottlerocket-metadata \
+  --migrate-to-version-from-os-release
 RemainAfterExit=true
 StandardError=journal+console
 
 [Install]
-WantedBy=multi-user.target
+RequiredBy=preconfigured.target
diff --git a/packages/os/os.spec b/packages/os/os.spec
@@ -34,6 +34,7 @@ Source107: [email protected]
 Source110: mark-successful-boot.service
 Source111: metricdog.service
 Source112: metricdog.timer
+Source113: send-boot-success.service
 
 # 2xx sources: tmpfilesd configs
 Source200: migration-tmpfiles.conf
@@ -337,7 +338,7 @@ install -p -m 0644 %{S:5} %{S:6} %{buildroot}%{_cross_templatedir}
 install -d %{buildroot}%{_cross_unitdir}
 install -p -m 0644 \
   %{S:100} %{S:101} %{S:102} %{S:103} %{S:105} \
-  %{S:106} %{S:107} %{S:110} %{S:111} %{S:112} \
+  %{S:106} %{S:107} %{S:110} %{S:111} %{S:112} %{S:113}\
   %{buildroot}%{_cross_unitdir}
 
 install -d %{buildroot}%{_cross_tmpfilesdir}
@@ -440,6 +441,7 @@ install -p -m 0644 %{S:300} %{buildroot}%{_cross_udevrulesdir}/80-ephemeral-stor
 %{_cross_templatedir}/metricdog-toml
 %{_cross_unitdir}/metricdog.service
 %{_cross_unitdir}/metricdog.timer
+%{_cross_unitdir}/send-boot-success.service
 
 %files -n %{_cross_os}logdog
 %{_cross_bindir}/logdog

diff --git a/packages/os/send-boot-success.service b/packages/os/send-boot-success.service
@@ -0,0 +1,16 @@
+[Unit]
+Description=Send boot success
+# The unit depends on 'configured.target' since Metricdog indirectly
+# depends on the proxy.env file created by settings-applier in the
+# preconfigured target
+After=network-online.target configured.target
+Wants=network-online.target configured.target
+
+[Service]
+Type=oneshot
+RemainAfterExit=true
+EnvironmentFile=/etc/network/proxy.env
+ExecStart=-/usr/bin/metricdog send-boot-success
+
+[Install]
+WantedBy=multi-user.target
diff --git a/packages/os/settings-applier.service b/packages/os/settings-applier.service
@@ -1,8 +1,13 @@
 [Unit]
 Description=Applies settings to create config files
 After=storewolf.service sundog.service early-boot-config.service apiserver.service
-Requires=apiserver.service
-Wants=storewolf.service sundog.service early-boot-config.service
+Requires=storewolf.service sundog.service early-boot-config.service
+# We don't want to restart the unit if apiserver restarts
+Wants=apiserver.service
+# Block manual interactions with this service, since it could leave the system in an
+# unexpected state
+RefuseManualStart=true
+RefuseManualStop=true
 
 [Service]
 Type=oneshot
@@ -12,4 +17,4 @@ RemainAfterExit=true
 StandardError=journal+console
 
 [Install]
-WantedBy=multi-user.target
+RequiredBy=preconfigured.target
diff --git a/packages/os/storewolf.service b/packages/os/storewolf.service
@@ -2,6 +2,10 @@
 Description=Datastore creator
 After=migrator.service
 Requires=migrator.service
+# Block manual interactions with this service, since it could leave the system in an
+# unexpected state
+RefuseManualStart=true
+RefuseManualStop=true
 
 [Service]
 Type=oneshot
@@ -10,4 +14,4 @@ RemainAfterExit=true
 StandardError=journal+console
 
 [Install]
-WantedBy=multi-user.target
+RequiredBy=preconfigured.target
diff --git a/packages/os/sundog.service b/packages/os/sundog.service
@@ -1,8 +1,14 @@
 [Unit]
 Description=User-specified setting generators
-# Need network access to support commands talking to IMDS.
+# Needs network access to support commands talking to IMDS.
 After=network-online.target apiserver.service early-boot-config.service
-Requires=network-online.target apiserver.service
+Requires=early-boot-config.service
+# We don't want to restart the unit if the network goes offline or apiserver restarts
+Wants=network-online.target apiserver.service
+# Block manual interactions with this service, since it could leave the system in an
+# unexpected state
+RefuseManualStart=true
+RefuseManualStop=true
 
 [Service]
 Type=oneshot
@@ -13,4 +19,4 @@ RemainAfterExit=true
 StandardError=journal+console
 
 [Install]
-WantedBy=multi-user.target
+RequiredBy=preconfigured.target
diff --git a/packages/release/activate-configured.service b/packages/release/activate-configured.service
@@ -0,0 +1,14 @@
+[Unit]
+Description=Isolates configured.target
+After=preconfigured.target
+Requires=preconfigured.target
+
+[Service]
+Type=oneshot
+ExecStart=/usr/bin/systemctl set-default configured
+ExecStart=/usr/bin/systemctl isolate default
+RemainAfterExit=true
+StandardError=journal+console
+
+[Install]
+WantedBy=preconfigured.target
diff --git a/packages/release/activate-multi-user.service b/packages/release/activate-multi-user.service
@@ -0,0 +1,14 @@
+[Unit]
+Description=Isolates multi-user.target
+After=configured.target
+Requires=configured.target
+
+[Service]
+Type=oneshot
+ExecStart=/usr/bin/systemctl set-default multi-user
+ExecStart=/usr/bin/systemctl isolate default
+RemainAfterExit=true
+StandardError=journal+console
+
+[Install]
+WantedBy=configured.target
diff --git a/packages/release/configured.target b/packages/release/configured.target
@@ -1,5 +1,8 @@
 [Unit]
-Description=Bottlerocket user and dynamic configuration complete
-After=settings-applier.service
-Requires=settings-applier.service early-boot-config.service
+Description=Bottlerocket final configuration complete
+After=preconfigured.target
+Requires=preconfigured.target
 AllowIsolate=yes
+
+[Install]
+RequiredBy=multi-user.target
diff --git a/packages/release/multi-user.target b/packages/release/multi-user.target
@@ -0,0 +1,7 @@
+[Unit]
+Description=Multi-User System
+Documentation=man:systemd.special(7)
+Requires=basic.target configured.target
+Conflicts=rescue.service rescue.target
+After=basic.target rescue.service rescue.target configured.target
+AllowIsolate=yes
diff --git a/packages/release/preconfigured.target b/packages/release/preconfigured.target
@@ -0,0 +1,11 @@
+[Unit]
+Description=Bottlerocket initial configuration complete
+AllowIsolate=yes
+After=basic.target
+Requires=basic.target
+# Prevent manually starting/stopping the target
+RefuseManualStart=true
+RefuseManualStop=true
+
+[Install]
+RequiredBy=configured.target multi-user.target
diff --git a/packages/release/release.spec b/packages/release/release.spec
@@ -16,7 +16,11 @@ Source200: motd.template
 Source201: proxy-env
 
 Source1000: eth0.xml
+Source1001: multi-user.target
 Source1002: configured.target
+Source1003: preconfigured.target
+Source1004: activate-configured.service
+Source1005: activate-multi-user.service
 
 # Mounts for writable local storage.
 Source1006: prepare-local.service
@@ -108,7 +112,8 @@ EOF
 
 install -d %{buildroot}%{_cross_unitdir}
 install -p -m 0644 \
-  %{S:1002} %{S:1006} %{S:1007} %{S:1008} %{S:1009} %{S:1010} %{S:1015} \
+  %{S:1001} %{S:1002} %{S:1003} %{S:1004} %{S:1005} \
+  %{S:1006} %{S:1007} %{S:1008} %{S:1009} %{S:1010} %{S:1015} \
   %{buildroot}%{_cross_unitdir}
 
 LOWERPATH=$(systemd-escape --path %{_cross_sharedstatedir}/kernel-devel/lower)
@@ -129,6 +134,8 @@ install -d %{buildroot}%{_cross_templatedir}
 install -p -m 0644 %{S:200} %{buildroot}%{_cross_templatedir}/motd
 install -p -m 0644 %{S:201} %{buildroot}%{_cross_templatedir}/proxy-env
 
+ln -s %{_cross_unitdir}/preconfigured.target %{buildroot}%{_cross_unitdir}/default.target
+
 %files
 %{_cross_factorydir}%{_cross_sysconfdir}/hosts
 %{_cross_factorydir}%{_cross_sysconfdir}/nsswitch.conf
@@ -138,6 +145,11 @@ install -p -m 0644 %{S:201} %{buildroot}%{_cross_templatedir}/proxy-env
 %{_cross_libdir}/os-release
 %{_cross_libdir}/systemd/system.conf.d/80-release.conf
 %{_cross_unitdir}/configured.target
+%{_cross_unitdir}/preconfigured.target
+%{_cross_unitdir}/multi-user.target
+%{_cross_unitdir}/default.target
+%{_cross_unitdir}/activate-configured.service
+%{_cross_unitdir}/activate-multi-user.service
 %{_cross_unitdir}/prepare-local.service
 %{_cross_unitdir}/var.mount
 %{_cross_unitdir}/opt.mount

diff --git a/packages/systemd/systemd.spec b/packages/systemd/systemd.spec
@@ -212,6 +212,13 @@ install -p -m 0644 %{S:3} %{buildroot}%{_cross_libdir}/systemd/journald.conf.d/j
 # with container networking by attempting to manage veth devices.
 rm -f %{buildroot}%{_cross_libdir}/systemd/network/*
 
+# Remove default, multi-user and graphical targets provided by systemd,
+# we override default/multi-user in the release spec and graphical
+# is never used
+rm -f %{buildroot}%{_cross_libdir}/systemd/{system,user}/default.target
+rm -f %{buildroot}%{_cross_libdir}/systemd/{system,user}/multi-user.target
+rm -f %{buildroot}%{_cross_libdir}/systemd/{system,user}/graphical.target
+
 %files
 %license LICENSE.GPL2 LICENSE.LGPL2.1
 %{_cross_attribution_file}
Original file line number	Diff line number	Diff line change
Expand Up		@@ -29,4 +29,3 @@ MemoryAccounting=true

		[Install]
		WantedBy=multi-user.target
		RequiredBy=mark-successful-boot.service