From 483f05324fb38ffdbf603e328422b85a4920f919 Mon Sep 17 00:00:00 2001
From: Andrew Battat <113942931+andrewbattat@users.noreply.github.com>
Date: Thu, 13 Feb 2025 10:54:51 -0600
Subject: [PATCH] chore(node): Processes robust against restarts (#3693)

NODE-1330
All critical services should restart on failure
---
 .../components/ic/generate-ic-config/generate-ic-config.service | 2 ++
 .../components/init/bootstrap-ic-node/bootstrap-ic-node.service | 2 ++
 .../guestos/generate-network-config.service                     | 2 ++
 .../hostos/generate-network-config.service                      | 2 ++
 4 files changed, 8 insertions(+)

diff --git a/ic-os/components/ic/generate-ic-config/generate-ic-config.service b/ic-os/components/ic/generate-ic-config/generate-ic-config.service
index 37a24722b64..1aa59c2706e 100644
--- a/ic-os/components/ic/generate-ic-config/generate-ic-config.service
+++ b/ic-os/components/ic/generate-ic-config/generate-ic-config.service
@@ -15,6 +15,8 @@ Wants=network-online.target
 [Service]
 Type=oneshot
 ExecStart=/opt/ic/bin/generate-ic-config.sh -n /boot/config/network.conf -c /boot/config/nns.conf -b /boot/config/backup.conf -m /boot/config/malicious_behavior.conf -q /boot/config/query_stats.conf -t /boot/config/jaeger_addr.conf -i /opt/ic/share/ic.json5.template -o /run/ic-node/config/ic.json5
+Restart=on-failure
+RestartSec=10
 
 [Install]
 WantedBy=multi-user.target
diff --git a/ic-os/components/init/bootstrap-ic-node/bootstrap-ic-node.service b/ic-os/components/init/bootstrap-ic-node/bootstrap-ic-node.service
index 1a72c1c1909..7de875f8e41 100644
--- a/ic-os/components/init/bootstrap-ic-node/bootstrap-ic-node.service
+++ b/ic-os/components/init/bootstrap-ic-node/bootstrap-ic-node.service
@@ -12,6 +12,8 @@ RequiredBy=setup-ssh-user-keys.service
 Type=oneshot
 RemainAfterExit=true
 ExecStart=/opt/ic/bin/bootstrap-ic-node.sh
+Restart=on-failure
+RestartSec=10
 
 # All guestos services that networking depends on log their outputs to the 
 # console to be piped to the host terminal if the verbose flag is enabled.
diff --git a/ic-os/components/networking/generate-network-config/guestos/generate-network-config.service b/ic-os/components/networking/generate-network-config/guestos/generate-network-config.service
index d52d4d12845..ac95c22640c 100644
--- a/ic-os/components/networking/generate-network-config/guestos/generate-network-config.service
+++ b/ic-os/components/networking/generate-network-config/guestos/generate-network-config.service
@@ -12,6 +12,8 @@ WantedBy=multi-user.target
 Type=oneshot
 RemainAfterExit=true
 ExecStart=/opt/ic/bin/guestos_tool generate-network-config
+Restart=on-failure
+RestartSec=10
 
 # All guestos services that networking depends on log their outputs to the 
 # console to be piped to the host terminal if the verbose flag is enabled.
diff --git a/ic-os/components/networking/generate-network-config/hostos/generate-network-config.service b/ic-os/components/networking/generate-network-config/hostos/generate-network-config.service
index 7973e91b276..cb357d6c030 100644
--- a/ic-os/components/networking/generate-network-config/hostos/generate-network-config.service
+++ b/ic-os/components/networking/generate-network-config/hostos/generate-network-config.service
@@ -11,6 +11,8 @@ RemainAfterExit=true
 ExecStart=/opt/ic/bin/hostos_tool generate-network-config
 StandardOutput=journal+console
 StandardError=journal+console
+Restart=on-failure
+RestartSec=10
 
 [Install]
 WantedBy=systemd-networkd.service