chore(node): Processes robust against restarts (#3693)

NODE-1330
All critical services should restart on failure

ic-os/components/ic/generate-ic-config/generate-ic-config.service

@@ -15,6 +15,8 @@ Wants=network-online.target
 [Service]
 Type=oneshot
 ExecStart=/opt/ic/bin/generate-ic-config.sh -n /boot/config/network.conf -c /boot/config/nns.conf -b /boot/config/backup.conf -m /boot/config/malicious_behavior.conf -q /boot/config/query_stats.conf -t /boot/config/jaeger_addr.conf -i /opt/ic/share/ic.json5.template -o /run/ic-node/config/ic.json5
+Restart=on-failure
+RestartSec=10
 
 [Install]
 WantedBy=multi-user.target

ic-os/components/init/bootstrap-ic-node/bootstrap-ic-node.service

@@ -12,6 +12,8 @@ RequiredBy=setup-ssh-user-keys.service
 Type=oneshot
 RemainAfterExit=true
 ExecStart=/opt/ic/bin/bootstrap-ic-node.sh
+Restart=on-failure
+RestartSec=10
 
 # All guestos services that networking depends on log their outputs to the 
 # console to be piped to the host terminal if the verbose flag is enabled.

ic-os/components/networking/generate-network-config/guestos/generate-network-config.service

@@ -12,6 +12,8 @@ WantedBy=multi-user.target
 Type=oneshot
 RemainAfterExit=true
 ExecStart=/opt/ic/bin/guestos_tool generate-network-config
+Restart=on-failure
+RestartSec=10
 
 # All guestos services that networking depends on log their outputs to the 
 # console to be piped to the host terminal if the verbose flag is enabled.

ic-os/components/networking/generate-network-config/hostos/generate-network-config.service

@@ -11,6 +11,8 @@ RemainAfterExit=true
 ExecStart=/opt/ic/bin/hostos_tool generate-network-config
 StandardOutput=journal+console
 StandardError=journal+console
+Restart=on-failure
+RestartSec=10
 
 [Install]
 WantedBy=systemd-networkd.service