Sequence startup and rewrite config (#494)

# Description

Make ceph services wait for microcephd and force a config rewrite on
startup. This should help avoid config inconsistencies and races.

## Type of change

Please delete options that are not relevant.

- [x] Bug fix (non-breaking change which fixes an issue)

## How Has This Been Tested?

Manual tests, standard CI testing

## Contributor's Checklist

Please check that you have:

- [x] self-reviewed the code in this PR.
- [x] added code comments, particularly in hard-to-understand areas.

Signed-off-by: Peter Sabaini <[email protected]>

microceph/ceph/start.go

@@ -148,6 +148,8 @@ func PostRefresh() error {
 
 // Start is run on daemon startup.
 func Start(ctx context.Context, s interfaces.StateInterface) error {
+	// flag: are we on the first run?
+	first := true
 	// Start background loop to refresh the config every minute if needed.
 	go func() {
 		oldMonitors := []string{}
@@ -182,8 +184,8 @@ func Start(ctx context.Context, s interfaces.StateInterface) error {
 				continue
 			}
 
-			// Compare to the previous list.
-			if reflect.DeepEqual(oldMonitors, monitors) {
+			// Check if we need to update
+			if !first || reflect.DeepEqual(oldMonitors, monitors) {
 				logger.Debugf("start: monitors unchanged, sleeping: %v", monitors)
 				time.Sleep(time.Minute)
 				continue
@@ -196,6 +198,7 @@ func Start(ctx context.Context, s interfaces.StateInterface) error {
 				continue
 			}
 			logger.Debug("start: updated config, sleeping")
+			first = false // for subsequent runs
 			oldMonitors = monitors
 			time.Sleep(time.Minute)
 		}

snap/snapcraft.yaml

@@ -61,13 +61,17 @@ apps:
     command: commands/mds.start
     daemon: simple
     install-mode: disable
+    after:
+      - daemon
     plugs:
       - network
       - network-bind
   mon:
     command: commands/mon.start
     daemon: simple
     install-mode: disable
+    after:
+      - daemon
     plugs:
       - hardware-observe
       - network
@@ -76,6 +80,8 @@ apps:
     command: commands/mgr.start
     daemon: simple
     install-mode: disable
+    after:
+      - daemon
     plugs:
       - network
       - network-bind
@@ -86,6 +92,8 @@ apps:
     install-mode: disable
     stop-mode: sigterm-all
     stop-timeout: 5m
+    after:
+      - daemon
     plugs:
       - block-devices
       - dm-crypt
@@ -98,6 +106,8 @@ apps:
     command: commands/rgw.start
     daemon: simple
     install-mode: disable
+    after:
+      - daemon
     plugs:
       - hardware-observe
       - network
@@ -107,6 +117,8 @@ apps:
     command: commands/rbd-mirror.start
     daemon: simple
     install-mode: disable
+    after:
+      - daemon
     plugs:
       - network
       - network-bind

snapcraft/commands/common

@@ -4,6 +4,7 @@ wait_for_config() {
   local max_attempts=300
   local attempt=0
 
+  sleep 1  # give microcephd startup a headstart
   while [ $attempt -lt $max_attempts ]; do
     if [ -f "${confpath}" ]; then
       if grep -q "${search_str}" "${confpath}"; then