ovn: fix corrupted database file on start (#3112)

dist/images/Dockerfile.base

@@ -28,6 +28,8 @@ RUN cd /usr/src/ && \
     curl -s https://github.com/kubeovn/ovs/commit/b721a251dadfb0857a2aebe32611d18ca935421f.patch | git apply && \
     # ovsdb-tool: add optional server id parameter for "join-cluster" command
     curl -s https://github.com/kubeovn/ovs/commit/0664cbe47d1684ca5b7147e3449779635454970e.patch | git apply && \
+    # ovsdb-tool: add command fix-cluster
+    curl -s https://github.com/kubeovn/ovs/commit/f52c239f5ded40b503e4d217f916b46ca413da4c.patch | git apply && \
     # fix memory leak in qos
     curl -s https://github.com/kubeovn/ovs/commit/6a4dd2f4b9311a227cc26fef7c398ae9b241311b.patch | git apply && \
     # compile without avx512

dist/images/start-db.sh

@@ -124,7 +124,14 @@ function ovn_db_pre_start() {
     ! ovsdb-tool db-is-clustered "$db_file" && return
     ovsdb-tool check-cluster "$db_file" && return
 
-    echo "detected database corruption for file $db_file, rebuild it."
+    local db_bak="$db_file.backup-$(date +%s)-$(random_str)"
+    echo "backup $db_file to $db_bak"
+    cp "$db_file" "$db_bak" || return 1
+
+    echo "detected database corruption for file $db_file, try to fix it."
+    ovsdb-tool fix-cluster "$db_file" && return
+
+    echo "failed to fix database file $db_file, rebuild it."
     local sid=$(ovsdb-tool db-sid "$db_file")
     if ! echo -n "$sid" | grep -qE '^[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}$'; then
         echo "failed to get sid from $1 db file $db_file"
@@ -148,10 +155,6 @@ function ovn_db_pre_start() {
     echo "generating new database file $db_new"
     ovsdb-tool --sid $sid join-cluster "$db_new" $db $local_addr ${remote_addr[*]} || return 1
 
-    local db_bak="$db_file.backup-$(random_str)"
-    echo "backup $db_file to $db_bak"
-    mv "$db_file" "$db_bak" || return 1
-
     echo "use new database file $db_new"
     mv "$db_new" "$db_file"
 }