Implement on_primary hooks.

This allows users of pg_auto_failover to setup their own scripts/actions to
complement a failover. The hooks are run in a process that is separate from
the main FSM, so as to prevent the system from making progress.

As a result, it's not possible for the hooks to change how things are
implemented in pg_auto_failover itself.

The hook system also allows running a user-defined "service", which is a
long running process or a deamon that belongs to pg_autoctl process tree.

 - [x] Implement a new internal service for running user-defined hooks
 - [ ] Implement pg_autoctl enable|disable run-hooks
 - [ ] Implement pg_autoctl create listener
 - [ ] Implement support for running script/commands (man system)
 - [ ] Add unit testing support for user-defined hooks
 - [ ] Document the new hook system, including tutorial
 - [ ] Add documentation examples covering pgbouncer as a hooked system

src/bin/pg_autoctl/cli_do_service.c

@@ -27,6 +27,7 @@
 #include "service_keeper.h"
 #include "service_monitor.h"
 #include "service_postgres_ctl.h"
+#include "service_run_hooks.h"
 #include "signals.h"
 #include "supervisor.h"
 
@@ -39,14 +40,17 @@ static void cli_do_service_getpid(const char *serviceName);
 static void cli_do_service_getpid_postgres(int argc, char **argv);
 static void cli_do_service_getpid_listener(int argc, char **argv);
 static void cli_do_service_getpid_node_active(int argc, char **argv);
+static void cli_do_service_getpid_run_hooks(int argc, char **argv);
 
 static void cli_do_service_restart(const char *serviceName);
 static void cli_do_service_restart_postgres(int argc, char **argv);
 static void cli_do_service_restart_listener(int argc, char **argv);
 static void cli_do_service_restart_node_active(int argc, char **argv);
+static void cli_do_service_restart_run_hooks(int argc, char **argv);
 
 static void cli_do_service_monitor_listener(int argc, char **argv);
 static void cli_do_service_node_active(int argc, char **argv);
+static void cli_do_service_run_hooks(int argc, char **argv);
 
 CommandLine service_pgcontroller =
 	make_command("pgcontroller",
@@ -80,6 +84,14 @@ CommandLine service_node_active =
 				 cli_getopt_pgdata,
 				 cli_do_service_node_active);
 
+CommandLine service_run_hooks =
+	make_command("run-hooks",
+				 "pg_autoctl service that run hooks (scripts)",
+				 CLI_PGDATA_USAGE,
+				 CLI_PGDATA_OPTION,
+				 cli_getopt_pgdata,
+				 cli_do_service_run_hooks);
+
 CommandLine service_getpid_postgres =
 	make_command("postgres",
 				 "Get the pid of the pg_autoctl postgres controller service",
@@ -104,10 +116,19 @@ CommandLine service_getpid_node_active =
 				 cli_getopt_pgdata,
 				 cli_do_service_getpid_node_active);
 
+CommandLine service_getpid_run_hooks =
+	make_command("run-hooks",
+				 "Get the pid of the pg_autoctl run-hooks service",
+				 CLI_PGDATA_USAGE,
+				 CLI_PGDATA_OPTION,
+				 cli_getopt_pgdata,
+				 cli_do_service_getpid_run_hooks);
+
 static CommandLine *service_getpid[] = {
 	&service_getpid_postgres,
 	&service_getpid_listener,
 	&service_getpid_node_active,
+	&service_getpid_run_hooks,
 	NULL
 };
 
@@ -141,10 +162,19 @@ CommandLine service_restart_node_active =
 				 cli_getopt_pgdata,
 				 cli_do_service_restart_node_active);
 
+CommandLine service_restart_run_hooks =
+	make_command("run-hooks",
+				 "Restart the pg_autoctl run-hooks service",
+				 CLI_PGDATA_USAGE,
+				 CLI_PGDATA_OPTION,
+				 cli_getopt_pgdata,
+				 cli_do_service_restart_run_hooks);
+
 static CommandLine *service_restart[] = {
 	&service_restart_postgres,
 	&service_restart_listener,
 	&service_restart_node_active,
+	&service_restart_run_hooks,
 	NULL
 };
 
@@ -160,6 +190,7 @@ static CommandLine *service[] = {
 	&service_postgres,
 	&service_monitor_listener,
 	&service_node_active,
+	&service_run_hooks,
 	NULL
 };
 
@@ -255,6 +286,16 @@ cli_do_service_getpid_node_active(int argc, char **argv)
 }
 
 
+/*
+ * cli_do_service_getpid_node_active gets the postgres service pid.
+ */
+static void
+cli_do_service_getpid_run_hooks(int argc, char **argv)
+{
+	(void) cli_do_service_getpid(SERVICE_NAME_RUN_HOOKS);
+}
+
+
 /*
  * cli_do_service_restart sends the TERM signal to the given serviceName, which
  * is known to have the restart policy RP_PERMANENT (that's hard-coded). As a
@@ -352,6 +393,18 @@ cli_do_service_restart_node_active(int argc, char **argv)
 }
 
 
+/*
+ * cli_do_service_restart_run_hooks sends the TERM signal to the run-hooks,
+ * which is known to have the restart policy RP_PERMANENT (that's hard-coded).
+ * As a consequence the supervisor will restart the service.
+ */
+static void
+cli_do_service_restart_run_hooks(int argc, char **argv)
+{
+	(void) cli_do_service_restart(SERVICE_NAME_RUN_HOOKS);
+}
+
+
 /*
  * cli_do_pgcontroller starts the process controller service within a supervision
  * tree. It is used for debug purposes only. When using this entry point we
@@ -588,3 +641,44 @@ cli_do_service_node_active(int argc, char **argv)
 	/* Start the node_active() protocol client */
 	(void) keeper_node_active_loop(&keeper, ppid);
 }
+
+
+/*
+ * cli_do_service_run_hooks starts the run-hooks service.
+ */
+static void
+cli_do_service_run_hooks(int argc, char **argv)
+{
+	Keeper keeper = { 0 };
+
+	pid_t ppid = getppid();
+
+	bool exitOnQuit = true;
+
+	keeper.config = keeperOptions;
+
+	/* Establish a handler for signals. */
+	(void) set_signal_handlers(exitOnQuit);
+
+	/* display a user-friendly process name */
+	(void) set_ps_title("pg_autoctl: run-hooks");
+
+	/* Prepare our Keeper and KeeperConfig from the CLI options */
+	if (!service_run_hooks_init(&keeper))
+	{
+		log_fatal("Failed to initialize the run-hooks service, "
+				  "see above for details");
+		exit(EXIT_CODE_INTERNAL_ERROR);
+	}
+
+	/* create the service pidfile */
+	if (!create_service_pidfile(keeper.config.pathnames.pid,
+								SERVICE_NAME_RUN_HOOKS))
+	{
+		/* errors have already been logged */
+		exit(EXIT_CODE_INTERNAL_ERROR);
+	}
+
+	/* Start the node_active() protocol client */
+	(void) service_run_hooks_loop(&keeper, ppid);
+}

src/bin/pg_autoctl/keeper_config.c

@@ -143,6 +143,18 @@
 	make_strbuf_option("replication", "backup_directory", NULL, \
 					   false, MAXPGPATH, config->backupDirectory)
 
+#define OPTION_HOOKS_ACTIVE(config) \
+	make_int_option_default("hooks", "active", NULL, \
+							false, &(config->enableHooks), 0)
+
+#define OPTION_HOOKS_ON_PRIMARY_CMD(config) \
+	make_strbuf_option("hooks", "on_primary", NULL, \
+					   false, BUFSIZE, config->onPrimaryCmd)
+
+#define OPTION_HOOKS_SERVICE_START_CMD(config) \
+	make_strbuf_option("hooks", "service", NULL, \
+					   false, BUFSIZE, config->serviceStartCmd)
+
 #define OPTION_TIMEOUT_NETWORK_PARTITION(config) \
 	make_int_option_default("timeout", "network_partition_timeout", \
 							NULL, false, \
@@ -241,6 +253,9 @@
 		OPTION_REPLICATION_MAXIMUM_BACKUP_RATE(config), \
 		OPTION_REPLICATION_BACKUP_DIR(config), \
 		OPTION_REPLICATION_PASSWORD(config), \
+		OPTION_HOOKS_ACTIVE(config), \
+		OPTION_HOOKS_ON_PRIMARY_CMD(config), \
+		OPTION_HOOKS_SERVICE_START_CMD(config), \
 		OPTION_TIMEOUT_NETWORK_PARTITION(config), \
 		OPTION_TIMEOUT_PREPARE_PROMOTION_CATCHUP(config), \
 		OPTION_TIMEOUT_PREPARE_PROMOTION_WALRECEIVER(config), \

src/bin/pg_autoctl/keeper_config.h

@@ -56,6 +56,11 @@ typedef struct KeeperConfig
 	char maximum_backup_rate[MAXIMUM_BACKUP_RATE_LEN];
 	char backupDirectory[MAXPGPATH];
 
+	/* Hooks settings (JSON strings) */
+	int enableHooks;            /* hooks.active */
+	char onPrimaryCmd[BUFSIZE];
+	char serviceStartCmd[BUFSIZE];
+
 	/* Citus specific options and settings */
 	char citusRoleStr[NAMEDATALEN];
 	CitusRole citusRole;

src/bin/pg_autoctl/service_keeper.c

@@ -27,6 +27,7 @@
 #include "pidfile.h"
 #include "service_keeper.h"
 #include "service_postgres_ctl.h"
+#include "service_run_hooks.h"
 #include "signals.h"
 #include "state.h"
 #include "string_utils.h"
@@ -85,6 +86,13 @@ start_keeper(Keeper *keeper)
 			-1,
 			&service_keeper_start,
 			(void *) keeper
+		},
+		{
+			SERVICE_NAME_RUN_HOOKS,
+			RP_PERMANENT,
+			-1,
+			&service_run_hooks_start,
+			(void *) keeper
 		}
 	};
 

src/bin/pg_autoctl/service_keeper_init.c

@@ -28,6 +28,7 @@
 #include "service_keeper.h"
 #include "service_keeper_init.h"
 #include "service_postgres_ctl.h"
+#include "service_run_hooks.h"
 #include "signals.h"
 #include "string_utils.h"
 #include "supervisor.h"
@@ -57,6 +58,13 @@ service_keeper_init(Keeper *keeper)
 			-1,
 			&service_keeper_init_start,
 			(void *) keeper
+		},
+		{
+			SERVICE_NAME_RUN_HOOKS,
+			RP_PERMANENT,
+			-1,
+			&service_run_hooks_start,
+			(void *) keeper
 		}
 	};