feat: add healthz check to orchestrator and to ui backend (#593)

* feat(healthz): refactor apiserver * feat(healthz): add healthz endpoint to orchestrator * feat(healthz): add healthz endpoint to ui backend * code review changes Co-authored-by: Krisztian Gacsal <[email protected]> --------- Co-authored-by: Krisztian Gacsal <[email protected]>
openclarity · Aug 25, 2023 · 4c56b1b · 4c56b1b
1 parent 723b18e
commit 4c56b1b
Show file tree

Hide file tree

Showing 9 changed files with 79 additions and 33 deletions.
diff --git a/cmd/vmclarity-apiserver/main.go b/cmd/vmclarity-apiserver/main.go
@@ -22,11 +22,8 @@ import (
 
 	"github.com/sirupsen/logrus"
 	"github.com/spf13/cobra"
-	"github.com/spf13/viper"
 
 	"github.com/openclarity/vmclarity/pkg/apiserver"
-	"github.com/openclarity/vmclarity/pkg/apiserver/config"
-	databaseTypes "github.com/openclarity/vmclarity/pkg/apiserver/database/types"
 	"github.com/openclarity/vmclarity/pkg/shared/log"
 	"github.com/openclarity/vmclarity/pkg/version"
 )
@@ -50,12 +47,6 @@ var (
 )
 
 func init() {
-	viper.SetDefault(config.HealthCheckAddress, ":8081")
-	viper.SetDefault(config.BackendRestPort, "8888")
-	viper.SetDefault(config.DatabaseDriver, databaseTypes.DBDriverTypeLocal)
-	viper.SetDefault(config.DisableOrchestrator, "false")
-	viper.AutomaticEnv()
-
 	cmdRun := cobra.Command{
 		Use:     "run",
 		Run:     runCommand,
@@ -91,7 +82,13 @@ func runCommand(_ *cobra.Command, _ []string) {
 	ctx := context.Background()
 	logger := logrus.WithContext(ctx)
 	ctx = log.SetLoggerForContext(ctx, logger)
-	apiserver.Run(ctx)
+
+	config, err := apiserver.LoadConfig()
+	if err != nil {
+		logger.Fatalf("failed to load API server config: %v", err)
+	}
+
+	apiserver.Run(ctx, config)
 }
 
 // Command to display the version.

diff --git a/cmd/vmclarity-orchestrator/main.go b/cmd/vmclarity-orchestrator/main.go
@@ -86,18 +86,16 @@ func runCommand(cmd *cobra.Command, _ []string) {
 	logger := logrus.WithContext(ctx)
 	ctx = log.SetLoggerForContext(ctx, logger)
 
-	orchestratorConfig, err := orchestrator.LoadConfig()
+	config, err := orchestrator.LoadConfig()
 	if err != nil {
 		logger.Fatalf("failed to load Orchestrator config: %v", err)
 	}
 
-	o, err := orchestrator.New(ctx, orchestratorConfig)
+	err = orchestrator.Run(ctx, config)
 	if err != nil {
-		logger.Fatalf("failed to initialize Orchestrator: %v", err)
+		logger.Fatalf("failed to run Orchestrator: %v", err)
 	}
 
-	o.Start(ctx)
-
 	// Wait for deactivation
 	sig := make(chan os.Signal, 1)
 	signal.Notify(sig, syscall.SIGHUP, syscall.SIGINT, syscall.SIGTERM, syscall.SIGQUIT)

diff --git a/cmd/vmclarity-ui-backend/main.go b/cmd/vmclarity-ui-backend/main.go
@@ -27,6 +27,8 @@ import (
 	"syscall"
 	"time"
 
+	"github.com/Portshift/go-utils/healthz"
+
 	"github.com/deepmap/oapi-codegen/pkg/middleware"
 	"github.com/labstack/echo/v4"
 	echomiddleware "github.com/labstack/echo/v4/middleware"
@@ -102,6 +104,10 @@ func runCommand(cmd *cobra.Command, _ []string) {
 		logger.Fatalf("unable to load configuration")
 	}
 
+	healthServer := healthz.NewHealthServer(config.HealthCheckAddress)
+	healthServer.Start()
+	healthServer.SetIsReady(false)
+
 	backendAddress := fmt.Sprintf("http://%s", net.JoinHostPort(config.APIServerHost, strconv.Itoa(config.APIServerPort)))
 	backendClient, err := backendclient.Create(backendAddress)
 	if err != nil {
@@ -127,6 +133,7 @@ func runCommand(cmd *cobra.Command, _ []string) {
 			logger.Fatalf("HTTP server shutdown %v", err)
 		}
 	}()
+	healthServer.SetIsReady(true)
 
 	// Wait for deactivation
 	sig := make(chan os.Signal, 1)

diff --git a/installation/docker/dockercompose.yml b/installation/docker/dockercompose.yml
@@ -26,12 +26,18 @@ services:
       - run
       - --log-level
       - info
+    ports:
+      - "8082:8082"
     env_file: ./orchestrator.env
     deploy:
       mode: replicated
       replicas: 1
       restart_policy:
         condition: on-failure
+    healthcheck:
+      test: wget --no-verbose --tries=1 --spider http://127.0.0.1:8082/healthz/ready || exit 1
+      interval: 10s
+      retries: 60
 
   ui:
     image: ${UIContainerImage:-ghcr.io/openclarity/vmclarity-ui:latest}
@@ -47,12 +53,18 @@ services:
       - run
       - --log-level
       - info
+    ports:
+      - "8083:8083"
     env_file: ./uibackend.env
     deploy:
       mode: replicated
       replicas: 1
       restart_policy:
         condition: on-failure
+    healthcheck:
+      test: wget --no-verbose --tries=1 --spider http://127.0.0.1:8083/healthz/ready || exit 1
+      interval: 10s
+      retries: 60
 
   gateway:
     image: nginx

diff --git a/pkg/apiserver/apiserver.go b/pkg/apiserver/apiserver.go
@@ -23,14 +23,13 @@ import (
 
 	"github.com/Portshift/go-utils/healthz"
 
-	_config "github.com/openclarity/vmclarity/pkg/apiserver/config"
 	"github.com/openclarity/vmclarity/pkg/apiserver/database"
 	databaseTypes "github.com/openclarity/vmclarity/pkg/apiserver/database/types"
 	"github.com/openclarity/vmclarity/pkg/apiserver/rest"
 	"github.com/openclarity/vmclarity/pkg/shared/log"
 )
 
-func createDatabaseConfig(config *_config.Config) databaseTypes.DBConfig {
+func createDatabaseConfig(config *Config) databaseTypes.DBConfig {
 	return databaseTypes.DBConfig{
 		DriverType:     config.DatabaseDriver,
 		EnableInfoLogs: config.EnableDBInfoLogs,
@@ -45,19 +44,13 @@ func createDatabaseConfig(config *_config.Config) databaseTypes.DBConfig {
 
 const defaultChanSize = 100
 
-func Run(ctx context.Context) {
+func Run(ctx context.Context, config *Config) {
 	logger := log.GetLoggerFromContextOrDiscard(ctx)
 
-	config, err := _config.LoadConfig()
-	if err != nil {
-		logger.Fatalf("Failed to load config: %v", err)
-	}
-
 	errChan := make(chan struct{}, defaultChanSize)
 
 	healthServer := healthz.NewHealthServer(config.HealthCheckAddress)
 	healthServer.Start()
-
 	healthServer.SetIsReady(false)
 
 	ctx, cancel := context.WithCancel(ctx)

diff --git a/pkg/apiserver/config/config.go → pkg/apiserver/config.go b/pkg/apiserver/config/config.go → pkg/apiserver/config.go
@@ -13,12 +13,14 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.
 
-package config
+package apiserver
 
 import (
 	"encoding/json"
 	"fmt"
 
+	databaseTypes "github.com/openclarity/vmclarity/pkg/apiserver/database/types"
+
 	log "github.com/sirupsen/logrus"
 	"github.com/spf13/viper"
 )
@@ -66,7 +68,18 @@ type Config struct {
 	LogLevel    log.Level `json:"log-level,omitempty"`
 }
 
+func setConfigDefaults() {
+	viper.SetDefault(HealthCheckAddress, ":8081")
+	viper.SetDefault(BackendRestPort, "8888")
+	viper.SetDefault(DatabaseDriver, databaseTypes.DBDriverTypeLocal)
+	viper.SetDefault(DisableOrchestrator, "false")
+
+	viper.AutomaticEnv()
+}
+
 func LoadConfig() (*Config, error) {
+	setConfigDefaults()
+
 	config := &Config{}
 
 	config.BackendRestHost = viper.GetString(BackendRestHost)

diff --git a/pkg/orchestrator/config.go b/pkg/orchestrator/config.go
@@ -36,6 +36,7 @@ const (
 	APIServerHost       = "APISERVER_HOST"
 	APIServerDisableTLS = "APISERVER_DISABLE_TLS"
 	APIServerPort       = "APISERVER_PORT"
+	HealthCheckAddress  = "HEALTH_CHECK_ADDRESS"
 
 	DeleteJobPolicy               = "DELETE_JOB_POLICY"
 	ScannerContainerImage         = "SCANNER_CONTAINER_IMAGE"
@@ -88,8 +89,9 @@ const (
 type Config struct {
 	ProviderKind models.CloudProvider
 
-	APIServerHost string `json:"apiserver-host,omitempty"`
-	APIServerPort int    `json:"apiserver-port,omitempty"`
+	APIServerHost      string `json:"apiserver-host,omitempty"`
+	APIServerPort      int    `json:"apiserver-port,omitempty"`
+	HealthCheckAddress string `json:"health-check-address,omitempty"`
 
 	// The Orchestrator starts the Controller(s) in a sequence and the ControllerStartupDelay is used for waiting
 	// before starting each Controller to avoid them hitting the API at the same time and allow one Controller
@@ -104,6 +106,7 @@ type Config struct {
 }
 
 func setConfigDefaults() {
+	viper.SetDefault(HealthCheckAddress, ":8082")
 	viper.SetDefault(DeleteJobPolicy, string(assetscanwatcher.DeleteJobPolicyAlways))
 	// https://github.com/openclarity/vmclarity-tools-base/blob/main/Dockerfile#L33
 	viper.SetDefault(GitleaksBinaryPath, "/artifacts/gitleaks")
@@ -167,6 +170,7 @@ func LoadConfig() (*Config, error) {
 	c := &Config{
 		APIServerHost:          apiServerHost,
 		APIServerPort:          apiServerPort,
+		HealthCheckAddress:     viper.GetString(HealthCheckAddress),
 		ProviderKind:           providerKind,
 		ControllerStartupDelay: viper.GetDuration(ControllerStartupDelay),
 		DiscoveryConfig: discovery.Config{

diff --git a/pkg/orchestrator/orchestrator.go b/pkg/orchestrator/orchestrator.go
@@ -22,6 +22,8 @@ import (
 	"strconv"
 	"time"
 
+	"github.com/Portshift/go-utils/healthz"
+
 	"github.com/openclarity/vmclarity/api/models"
 	"github.com/openclarity/vmclarity/pkg/orchestrator/assetscanprocessor"
 	"github.com/openclarity/vmclarity/pkg/orchestrator/assetscanwatcher"
@@ -45,6 +47,22 @@ type Orchestrator struct {
 	controllerStartupDelay time.Duration
 }
 
+func Run(ctx context.Context, config *Config) error {
+	healthServer := healthz.NewHealthServer(config.HealthCheckAddress)
+	healthServer.Start()
+	healthServer.SetIsReady(false)
+
+	orchestrator, err := New(ctx, config)
+	if err != nil {
+		return fmt.Errorf("failed to initialize Orchestrator: %w", err)
+	}
+
+	orchestrator.Start(ctx)
+	healthServer.SetIsReady(true)
+
+	return nil
+}
+
 // NewWithProvider returns an Orchestrator initialized using the p provider.Provider.
 // Use this method when Orchestrator needs to rely on custom provider.Provider implementation.
 // E.g. End-to-End testing.

diff --git a/pkg/uibackend/config.go b/pkg/uibackend/config.go
@@ -24,23 +24,27 @@ const (
 	APIServerHost       = "APISERVER_HOST"
 	APIServerDisableTLS = "APISERVER_DISABLE_TLS"
 	APIServerPort       = "APISERVER_PORT"
+	HealthCheckAddress  = "HEALTH_CHECK_ADDRESS"
 )
 
 type Config struct {
-	ListenAddress string `json:"listen-address,omitempty"`
-	APIServerHost string `json:"apiserver-host,omitempty"`
-	APIServerPort int    `json:"apiserver-port,omitempty"`
+	ListenAddress      string `json:"listen-address,omitempty"`
+	APIServerHost      string `json:"apiserver-host,omitempty"`
+	APIServerPort      int    `json:"apiserver-port,omitempty"`
+	HealthCheckAddress string `json:"health-check-address,omitempty"`
 }
 
 func LoadConfig() (*Config, error) {
 	viper.AutomaticEnv()
 
 	viper.SetDefault(ListenAddress, ":8890")
+	viper.SetDefault(HealthCheckAddress, ":8083")
 
 	c := &Config{
-		ListenAddress: viper.GetString(ListenAddress),
-		APIServerHost: viper.GetString(APIServerHost),
-		APIServerPort: viper.GetInt(APIServerPort),
+		ListenAddress:      viper.GetString(ListenAddress),
+		APIServerHost:      viper.GetString(APIServerHost),
+		APIServerPort:      viper.GetInt(APIServerPort),
+		HealthCheckAddress: viper.GetString(HealthCheckAddress),
 	}
 	return c, nil
 }