feat: process all config fragments as golang templates (#114)

### Description

OB-37310 Process all config fragments as golang templates

Replaces #100 and #111

### Checklist
- [x] Created tests which fail without the change (if possible)
- [ ] Extended the README / documentation, if necessary

Co-authored-by: Alex Lew <[email protected]>

go.mod

@@ -11,8 +11,10 @@ require (
 	github.com/spf13/viper v1.20.0-alpha.6
 	github.com/stretchr/testify v1.9.0
 	go.opentelemetry.io/collector/otelcol v0.110.0
+	go.uber.org/zap v1.27.0
 	golang.org/x/sys v0.25.0
 	gopkg.in/yaml.v2 v2.4.0
+	gopkg.in/yaml.v3 v3.0.1
 )
 
 require (
@@ -337,7 +339,6 @@ require (
 	go.opentelemetry.io/proto/otlp v1.3.1 // indirect
 	go.uber.org/atomic v1.11.0 // indirect
 	go.uber.org/multierr v1.11.0 // indirect
-	go.uber.org/zap v1.27.0 // indirect
 	golang.org/x/crypto v0.27.0 // indirect
 	golang.org/x/exp v0.0.0-20240613232115-7f521ea00fb8 // indirect
 	golang.org/x/mod v0.19.0 // indirect
@@ -357,7 +358,6 @@ require (
 	gopkg.in/inf.v0 v0.9.1 // indirect
 	gopkg.in/ini.v1 v1.67.0 // indirect
 	gopkg.in/natefinch/lumberjack.v2 v2.2.1 // indirect
-	gopkg.in/yaml.v3 v3.0.1 // indirect
 	k8s.io/api v0.31.1 // indirect
 	k8s.io/apimachinery v0.31.1 // indirect
 	k8s.io/client-go v0.31.1 // indirect

internal/commands/start/start.go

@@ -4,37 +4,61 @@ Copyright © 2024 NAME HERE <EMAIL ADDRESS>
 package start
 
 import (
+	"context"
 	"os"
 
+	logger "github.com/observeinc/observe-agent/internal/commands/util"
 	"github.com/observeinc/observe-agent/internal/config"
+	"github.com/observeinc/observe-agent/internal/connections"
 	"github.com/observeinc/observe-agent/internal/root"
 	"github.com/observeinc/observe-agent/observecol"
 	"github.com/spf13/cobra"
 	"github.com/spf13/viper"
+	collector "go.opentelemetry.io/collector/otelcol"
 )
 
+func SetupAndGenerateCollectorSettings() (*collector.CollectorSettings, func(), error) {
+	ctx := logger.WithCtx(context.Background(), logger.Get())
+	// Set Env Vars from config
+	err := config.SetEnvVars()
+	if err != nil {
+		return nil, nil, err
+	}
+	// Set up our temp dir annd temp config files
+	tmpDir, err := os.MkdirTemp("", connections.TempFilesFolder)
+	if err != nil {
+		return nil, nil, err
+	}
+	configFilePaths, overridePath, err := config.GetAllOtelConfigFilePaths(ctx, tmpDir)
+	cleanup := func() {
+		if overridePath != "" {
+			os.Remove(overridePath)
+		}
+		os.RemoveAll(tmpDir)
+	}
+	if err != nil {
+		cleanup()
+		return nil, nil, err
+	}
+	// Generate collector settings with all config files
+	colSettings := observecol.GenerateCollectorSettings(configFilePaths)
+	return colSettings, cleanup, nil
+}
+
 var startCmd = &cobra.Command{
 	Use:   "start",
 	Short: "Start the Observe agent process.",
 	Long: `The Observe agent is based on the OpenTelemetry Collector. 
 This command reads in the local config and env vars and starts the 
 collector on the current host.`,
 	RunE: func(cmd *cobra.Command, args []string) error {
-		// Set Env Vars from config
-		err := config.SetEnvVars()
+		colSettings, cleanup, err := SetupAndGenerateCollectorSettings()
 		if err != nil {
 			return err
 		}
-		//
-		configFilePaths, overridePath, err := config.GetAllOtelConfigFilePaths()
-		if err != nil {
-			return err
-		}
-		if overridePath != "" {
-			defer os.Remove(overridePath)
+		if cleanup != nil {
+			defer cleanup()
 		}
-		// Generate collector settings with all config files
-		colSettings := observecol.GenerateCollectorSettings(configFilePaths)
 		otelCmd := observecol.GetOtelCollectorCommand(colSettings)
 		return otelCmd.RunE(cmd, args)
 	},

internal/commands/util/logger.go

@@ -0,0 +1,37 @@
+package logger
+
+import (
+	"context"
+
+	"go.uber.org/zap"
+)
+
+type ctxKey struct{}
+
+func Get() *zap.Logger {
+	logger, _ := zap.NewProduction()
+	return logger
+}
+
+func GetDev() *zap.Logger {
+	logger, _ := zap.NewDevelopment()
+	return logger
+}
+
+func FromCtx(ctx context.Context) *zap.Logger {
+	if l, ok := ctx.Value(ctxKey{}).(*zap.Logger); ok {
+		return l
+	}
+
+	return Get()
+}
+
+func WithCtx(ctx context.Context, l *zap.Logger) context.Context {
+	if lp, ok := ctx.Value(ctxKey{}).(*zap.Logger); ok {
+		if lp == l {
+			return ctx
+		}
+	}
+
+	return context.WithValue(ctx, ctxKey{}, l)
+}

internal/config/confighandler.go

@@ -1,6 +1,7 @@
 package config
 
 import (
+	"context"
 	"fmt"
 	"net/url"
 	"os"
@@ -11,14 +12,18 @@ import (
 	"github.com/spf13/viper"
 )
 
-func GetAllOtelConfigFilePaths() ([]string, string, error) {
+func GetAllOtelConfigFilePaths(ctx context.Context, tmpDir string) ([]string, string, error) {
 	// Initialize config file paths with base config
 	configFilePaths := []string{filepath.Join(GetDefaultConfigFolder(), "otel-collector.yaml")}
 	var err error
 	// Get additional config paths based on connection configs
 	for _, conn := range connections.AllConnectionTypes {
 		if viper.IsSet(conn.Name) {
-			configFilePaths = append(configFilePaths, conn.GetConfigFilePaths()...)
+			connectionPaths, err := conn.GetConfigFilePaths(ctx, tmpDir)
+			if err != nil {
+				return nil, "", err
+			}
+			configFilePaths = append(configFilePaths, connectionPaths...)
 		}
 	}
 	// Read in otel-config flag and add to paths if set

internal/connections/connections.go

@@ -1,13 +1,20 @@
 package connections
 
 import (
+	"context"
+	"fmt"
 	"os"
 	"path/filepath"
 	"runtime"
+	"text/template"
 
+	logger "github.com/observeinc/observe-agent/internal/commands/util"
 	"github.com/spf13/viper"
+	"go.uber.org/zap"
 )
 
+var TempFilesFolder = "observe-agent"
+
 type ConfigFieldHandler interface {
 	GenerateCollectorConfigFragment() interface{}
 }
@@ -20,6 +27,10 @@ type CollectorConfigFragment struct {
 type ConnectionType struct {
 	Name         string
 	ConfigFields []CollectorConfigFragment
+	Type         string
+
+	configFolderPath string
+	getConfig        func() *viper.Viper
 }
 
 func GetConfigFolderPath() string {
@@ -39,23 +50,111 @@ func GetConfigFolderPath() string {
 	}
 }
 
-func (c ConnectionType) GetConfigFilePaths() []string {
-	var rawConnConfig = viper.Sub(c.Name)
-	configPaths := make([]string, 0)
-	if rawConnConfig == nil || !rawConnConfig.GetBool("enabled") {
-		return configPaths
+func (c *ConnectionType) GetTemplateFilepath(tplFilename string) string {
+	return filepath.Join(c.configFolderPath, c.Name, tplFilename)
+}
+
+func (c *ConnectionType) RenderConfigTemplate(ctx context.Context, tmpDir string, tplFilename string, confValues any) (string, error) {
+	tplPath := c.GetTemplateFilepath(tplFilename)
+	tmpl, err := template.New("").Funcs(GetTemplateFuncMap()).ParseFiles(tplPath)
+	if err != nil {
+		logger.FromCtx(ctx).Error("failed to parse config fragment template", zap.String("file", tplPath), zap.Error(err))
+		return "", err
 	}
+	f, err := os.CreateTemp(tmpDir, fmt.Sprintf("*-%s", tplFilename))
+	if err != nil {
+		logger.FromCtx(ctx).Error("failed to create temporary config fragment file", zap.String("file", tplPath), zap.Error(err))
+		return "", err
+	}
+	err = tmpl.ExecuteTemplate(f, tplFilename, confValues)
+	if err != nil {
+		logger.FromCtx(ctx).Error("failed to execute config fragment template", zap.String("file", tplPath), zap.Error(err))
+		return "", err
+	}
+	return f.Name(), nil
+}
+
+func (c *ConnectionType) ProcessConfigFields(ctx context.Context, tmpDir string, rawConnConfig *viper.Viper, confValues any) ([]string, error) {
+	paths := make([]string, 0)
 	for _, field := range c.ConfigFields {
 		val := rawConnConfig.GetBool(field.configYAMLPath)
 		if val && field.colConfigFilePath != "" {
-			configPath := filepath.Join(GetConfigFolderPath(), c.Name, field.colConfigFilePath)
-			configPaths = append(configPaths, configPath)
+			configPath, err := c.RenderConfigTemplate(ctx, tmpDir, field.colConfigFilePath, confValues)
+			if err != nil {
+				return nil, err
+			}
+			paths = append(paths, configPath)
+		}
+	}
+	return paths, nil
+}
+
+func (c *ConnectionType) GetConfigFilePaths(ctx context.Context, tmpDir string) ([]string, error) {
+	var rawConnConfig = c.getConfig()
+	var configPaths []string
+	if rawConnConfig == nil || !rawConnConfig.GetBool("enabled") {
+		return configPaths, nil
+	}
+	switch c.Type {
+	case SelfMonitoringConnectionTypeName:
+		conf := &SelfMonitoringConfig{}
+		err := rawConnConfig.Unmarshal(conf)
+		if err != nil {
+			logger.FromCtx(ctx).Error("failed to unmarshal config", zap.String("connection", c.Name))
+			return nil, err
+		}
+		configPaths, err = c.ProcessConfigFields(ctx, tmpDir, rawConnConfig, conf)
+		if err != nil {
+			return nil, err
 		}
+	case HostMonitoringConnectionTypeName:
+		conf := &HostMonitoringConfig{}
+		err := rawConnConfig.Unmarshal(conf)
+		if err != nil {
+			logger.FromCtx(ctx).Error("failed to unmarshal config", zap.String("connection", c.Name))
+			return nil, err
+		}
+		configPaths, err = c.ProcessConfigFields(ctx, tmpDir, rawConnConfig, conf)
+		if err != nil {
+			return nil, err
+		}
+	default:
+		logger.FromCtx(ctx).Error("unknown connection type", zap.String("type", c.Type))
+		return nil, fmt.Errorf("unknown connection type %s", c.Type)
+	}
+	return configPaths, nil
+}
+
+type ConnectionTypeOption func(*ConnectionType)
+
+func MakeConnectionType(Name string, ConfigFields []CollectorConfigFragment, Type string, opts ...ConnectionTypeOption) *ConnectionType {
+	var c = &ConnectionType{Name: Name, ConfigFields: ConfigFields, Type: Type}
+	c.getConfig = func() *viper.Viper {
+		return viper.Sub(c.Name)
+	}
+	c.configFolderPath = GetConfigFolderPath()
+
+	// Apply provided options
+	for _, opt := range opts {
+		opt(c)
+	}
+
+	return c
+}
+
+func WithConfigFolderPath(configFolderPath string) ConnectionTypeOption {
+	return func(c *ConnectionType) {
+		c.configFolderPath = configFolderPath
+	}
+}
+
+func WithGetConfig(getConfig func() *viper.Viper) ConnectionTypeOption {
+	return func(c *ConnectionType) {
+		c.getConfig = getConfig
 	}
-	return configPaths
 }
 
 var AllConnectionTypes = []*ConnectionType{
-	&HostMonitoringConnectionType,
-	&SelfMonitoringConnectionType,
+	HostMonitoringConnectionType,
+	SelfMonitoringConnectionType,
 }