diff --git a/cmd/installer-downloader/main.go b/cmd/installer-downloader/main.go index 45c628dc13069..5ddadc54c0d60 100644 --- a/cmd/installer-downloader/main.go +++ b/cmd/installer-downloader/main.go @@ -72,6 +72,7 @@ func runDownloader(ctx context.Context, env *env.Env, version string, flavor str return fmt.Errorf("failed to download installer: %w", err) } cmd := exec.CommandContext(ctx, filepath.Join(tmpDir, installerBinPath), "setup", "--flavor", flavor) + cmd.Dir = tmpDir cmd.Stdout = os.Stdout cmd.Stderr = os.Stderr cmd.Env = append(os.Environ(), telemetry.EnvFromContext(ctx)...) @@ -92,6 +93,10 @@ func downloadInstaller(ctx context.Context, env *env.Env, version string, tmpDir if downloadedPackage.Name != installerPackage { return fmt.Errorf("unexpected package name: %s, expected %s", downloadedPackage.Name, installerPackage) } + err = downloadedPackage.WriteOCILayout(tmpDir) + if err != nil { + return fmt.Errorf("failed to write OCI layout: %w", err) + } err = downloadedPackage.ExtractLayers(oci.DatadogPackageLayerMediaType, tmpDir) if err != nil { return fmt.Errorf("failed to extract layers: %w", err) diff --git a/pkg/fleet/installer/packages/datadog_agent.go b/pkg/fleet/installer/packages/datadog_agent.go index 7236096bbf1e1..2b7d4192d4592 100644 --- a/pkg/fleet/installer/packages/datadog_agent.go +++ b/pkg/fleet/installer/packages/datadog_agent.go @@ -104,7 +104,7 @@ func SetupAgent(ctx context.Context, _ []string) (err error) { if err = os.MkdirAll("/etc/datadog-agent", 0755); err != nil { return fmt.Errorf("failed to create /etc/datadog-agent: %v", err) } - ddAgentUID, ddAgentGID, err := GetAgentIDs() + ddAgentUID, ddAgentGID, err := getAgentIDs() if err != nil { return fmt.Errorf("error getting dd-agent user and group IDs: %w", err) } @@ -230,7 +230,7 @@ func chownRecursive(path string, uid int, gid int, ignorePaths []string) error { // StartAgentExperiment starts the agent experiment func StartAgentExperiment(ctx context.Context) error { - ddAgentUID, ddAgentGID, err := GetAgentIDs() + ddAgentUID, ddAgentGID, err := getAgentIDs() if err != nil { return fmt.Errorf("error getting dd-agent user and group IDs: %w", err) } diff --git a/pkg/fleet/installer/packages/datadog_installer.go b/pkg/fleet/installer/packages/datadog_installer.go index ceaf08bf330d6..14d0f042cd0e6 100644 --- a/pkg/fleet/installer/packages/datadog_installer.go +++ b/pkg/fleet/installer/packages/datadog_installer.go @@ -60,7 +60,7 @@ func SetupInstaller(ctx context.Context) (err error) { if err != nil { return fmt.Errorf("error adding dd-agent user to dd-agent group: %w", err) } - ddAgentUID, ddAgentGID, err := GetAgentIDs() + ddAgentUID, ddAgentGID, err := getAgentIDs() if err != nil { return fmt.Errorf("error getting dd-agent user and group IDs: %w", err) } @@ -165,8 +165,8 @@ func SetupInstaller(ctx context.Context) (err error) { return startInstallerStable(ctx) } -// GetAgentIDs returns the UID and GID of the dd-agent user and group. -func GetAgentIDs() (uid, gid int, err error) { +// getAgentIDs returns the UID and GID of the dd-agent user and group. +func getAgentIDs() (uid, gid int, err error) { ddAgentUser, err := user.Lookup("dd-agent") if err != nil { return -1, -1, fmt.Errorf("dd-agent user not found: %w", err) diff --git a/pkg/fleet/installer/setup/common/config.go b/pkg/fleet/installer/setup/common/config.go index 865b617c6283a..fb1677aeb6964 100644 --- a/pkg/fleet/installer/setup/common/config.go +++ b/pkg/fleet/installer/setup/common/config.go @@ -3,242 +3,211 @@ // This product includes software developed at Datadog (https://www.datadoghq.com/). // Copyright 2016-present Datadog, Inc. -//go:build !windows - -// Package common contains the HostInstaller struct which is used to write the agent agentConfiguration to disk package common import ( - "context" "fmt" "os" "path/filepath" - "github.com/DataDog/datadog-agent/pkg/fleet/installer/env" - "github.com/DataDog/datadog-agent/pkg/fleet/installer/oci" - "github.com/DataDog/datadog-agent/pkg/fleet/installer/packages" - "github.com/DataDog/datadog-agent/pkg/fleet/internal/exec" - "gopkg.in/yaml.v2" + "gopkg.in/yaml.v3" ) -var ( +const ( + disclaimerGenerated = `# This file was generated by the Datadog Installer. +# Other configuration options are available, see https://docs.datadoghq.com/agent/guide/agent-configuration-files/ for more information.` + configDir = "/etc/datadog-agent" - datadogConfFile = filepath.Join(configDir, "datadog.yaml") - logsConfFile = filepath.Join(configDir, "conf.d/configured_at_install_logs.yaml") - sparkConfigFile = filepath.Join(configDir, "conf.d/spark.d/spark.yaml") - injectTracerConfigFile = filepath.Join(configDir, "inject/tracer.yaml") + datadogConfFile = "datadog.yaml" + injectTracerConfigFile = "inject/tracer.yaml" ) -// HostInstaller is a struct that represents the agent agentConfiguration -// used to write the agentConfiguration to disk in datadog-installer custom setup scenarios -type HostInstaller struct { - env *env.Env - - agentConfig map[string]interface{} - logsConfig logsConfig - sparkConfig sparkConfig - injectorConfig injectorConfig - hostTags []tag - ddUID int - ddGID int - - injectorVersion string - javaVersion string - agentVersion string -} - -type tag struct { - key string `yaml:"key"` - value string `yaml:"value"` -} - -type logsConfig struct { - Logs []LogConfig `yaml:"logs"` -} - -// LogConfig is a struct that represents a single log agentConfiguration -type LogConfig struct { - Type string `yaml:"type"` - Path string `yaml:"path"` - Service string `yaml:"service,omitempty"` - Source string `yaml:"source,omitempty"` -} - -type sparkConfig struct { - InitConfig interface{} `yaml:"init_config,omitempty"` - Instances []SparkInstance `yaml:"instances"` -} - -// SparkInstance is a struct that represents a single spark instance -type SparkInstance struct { - SparkURL string `yaml:"spark_url"` - SparkClusterMode string `yaml:"spark_cluster_mode,omitempty"` - ClusterName string `yaml:"cluster_name"` - StreamingMetrics bool `yaml:"streaming_metrics,omitempty"` -} - -type injectorConfig struct { - Version int `yaml:"version"` - ConfigSources string `yaml:"config_sources"` - EnvsToInject []EnvVar `yaml:"additional_environment_variables"` -} - -// EnvVar is a struct that represents an environment variable -type EnvVar struct { - Key string `yaml:"key"` - Value string `yaml:"value"` -} - -// NewHostInstaller creates a new HostInstaller struct and loads the existing agentConfiguration from disk -func NewHostInstaller(env *env.Env) (*HostInstaller, error) { - ddUID, ddGID, err := packages.GetAgentIDs() +func writeConfigs(config Config, configDir string) error { + err := writeConfig(filepath.Join(configDir, datadogConfFile), config.DatadogYAML, 0640, true) if err != nil { - return nil, fmt.Errorf("failed to get agent user and group IDs: %v", err) + return fmt.Errorf("could not write datadog.yaml: %w", err) } - return newHostInstaller(env, ddUID, ddGID) + err = writeConfig(filepath.Join(configDir, injectTracerConfigFile), config.InjectTracerYAML, 0644, false) + if err != nil { + return fmt.Errorf("could not write tracer.yaml: %w", err) + } + for name, config := range config.IntegrationConfigs { + err = writeConfig(filepath.Join(configDir, "conf.d", name), config, 0644, false) + if err != nil { + return fmt.Errorf("could not write %s.yaml: %w", name, err) + } + } + return nil } -func newHostInstaller(env *env.Env, ddUID, ddGID int) (*HostInstaller, error) { - i := &HostInstaller{agentConfig: make(map[string]interface{})} - if env.APIKey == "" { - return nil, fmt.Errorf("DD_API key is required") +func writeConfig(path string, config any, perms os.FileMode, merge bool) error { + serializedNewConfig, err := yaml.Marshal(config) + if err != nil { + return fmt.Errorf("could not serialize config: %w", err) } - i.AddAgentConfig("api_key", env.APIKey) - - if env.Site != "" { - i.AddAgentConfig("site", env.Site) + var newConfig map[string]interface{} + err = yaml.Unmarshal(serializedNewConfig, &newConfig) + if err != nil { + return fmt.Errorf("could not unmarshal config: %w", err) + } + if len(newConfig) == 0 { + return nil + } + err = os.MkdirAll(filepath.Dir(path), 0755) + if err != nil { + return fmt.Errorf("could not create config directory: %w", err) } - i.ddUID = ddUID - i.ddGID = ddGID - i.env = env - return i, nil + var existingConfig map[string]interface{} + if merge { + serializedExistingConfig, err := os.ReadFile(path) + if err != nil && !os.IsNotExist(err) { + return fmt.Errorf("could not read existing config: %w", err) + } + if err == nil { + err = yaml.Unmarshal(serializedExistingConfig, &existingConfig) + if err != nil { + return fmt.Errorf("could not unmarshal existing config: %w", err) + } + } + } + merged, err := mergeConfig(existingConfig, newConfig) + if err != nil { + return fmt.Errorf("could not merge config: %w", err) + } + serializedMerged, err := yaml.Marshal(merged) + if err != nil { + return fmt.Errorf("could not serialize merged config: %w", err) + } + if len(existingConfig) == 0 { + serializedMerged = []byte(disclaimerGenerated + "\n\n" + string(serializedMerged)) + } + err = os.WriteFile(path, serializedMerged, perms) + if err != nil { + return fmt.Errorf("could not write config: %w", err) + } + return nil } -// SetAgentVersion sets the agent version to install -func (i *HostInstaller) SetAgentVersion(version string) { - i.agentVersion = version +// Config represents the configuration to write in /etc/datadog-agent +type Config struct { + // DatadogYAML is the content of the datadog.yaml file + DatadogYAML DatadogConfig + // InjectTracerYAML is the content of the inject/tracer.yaml file + InjectTracerYAML InjectTracerConfig + // IntegrationConfigs is the content of the integration configuration files under conf.d/ + IntegrationConfigs map[string]IntegrationConfig } -// SetInjectorVersion sets the injector version to install -func (i *HostInstaller) SetInjectorVersion(version string) { - i.injectorVersion = version +// DatadogConfig represents the configuration to write in /etc/datadog-agent/datadog.yaml +type DatadogConfig struct { + APIKey string `yaml:"api_key"` + Hostname string `yaml:"hostname,omitempty"` + Site string `yaml:"site,omitempty"` + Env string `yaml:"env,omitempty"` + Tags []string `yaml:"tags,omitempty"` + LogsEnabled bool `yaml:"logs_enabled,omitempty"` + DJM DatadogConfigDJM `yaml:"djm,omitempty"` + ProcessConfig DatadogConfigProcessConfig `yaml:"process_config,omitempty"` + ExpectedTagsDuration string `yaml:"expected_tags_duration,omitempty"` } -// SetJavaTracerVersion sets the java tracer version to install -func (i *HostInstaller) SetJavaTracerVersion(version string) { - i.javaVersion = version +// DatadogConfigDJM represents the configuration for the Data Jobs Monitoring +type DatadogConfigDJM struct { + Enabled bool `yaml:"enabled,omitempty"` } -// AddTracerEnv adds an environment variable to the list of environment variables to inject -func (i *HostInstaller) AddTracerEnv(key, value string) { - i.injectorConfig.EnvsToInject = append(i.injectorConfig.EnvsToInject, EnvVar{Key: key, Value: value}) +// DatadogConfigProcessConfig represents the configuration for the process agent +type DatadogConfigProcessConfig struct { + ExpvarPort int `yaml:"expvar_port,omitempty"` } -// AddAgentConfig adds a key value pair to the agent agentConfiguration -func (i *HostInstaller) AddAgentConfig(key string, value interface{}) { - i.agentConfig[key] = value +// IntegrationConfig represents the configuration for an integration under conf.d/ +type IntegrationConfig struct { + InitConfig []any `yaml:"init_config,omitempty"` + Instances []any `yaml:"instances,omitempty"` + Logs []IntegrationConfigLogs `yaml:"logs,omitempty"` } -// AddLogConfig adds a log agentConfiguration to the agent configuration -func (i *HostInstaller) AddLogConfig(log LogConfig) { - i.logsConfig.Logs = append(i.logsConfig.Logs, log) - if len(i.logsConfig.Logs) == 1 { - i.AddAgentConfig("logs_enabled", true) - } +// IntegrationConfigLogs represents the configuration for the logs of an integration +type IntegrationConfigLogs struct { + Type string `yaml:"type,omitempty"` + Path string `yaml:"path,omitempty"` + Service string `yaml:"service,omitempty"` + Source string `yaml:"source,omitempty"` } -// AddSparkInstance adds a spark instance to the agent agentConfiguration -func (i *HostInstaller) AddSparkInstance(spark SparkInstance) { - i.sparkConfig.Instances = append(i.sparkConfig.Instances, spark) +// IntegrationConfigInstanceSpark represents the configuration for the Spark integration +type IntegrationConfigInstanceSpark struct { + SparkURL string `yaml:"spark_url"` + SparkClusterMode string `yaml:"spark_cluster_mode"` + ClusterName string `yaml:"cluster_name"` + StreamingMetrics bool `yaml:"streaming_metrics"` } -// AddHostTag adds a host tag to the agent agentConfiguration -func (i *HostInstaller) AddHostTag(key, value string) { - i.hostTags = append(i.hostTags, tag{key, value}) +// InjectTracerConfig represents the configuration to write in /etc/datadog-agent/inject/tracer.yaml +type InjectTracerConfig struct { + Version int `yaml:"version,omitempty"` + ConfigSources string `yaml:"config_sources,omitempty"` + AdditionalEnvironmentVariables []InjectTracerConfigEnvVar `yaml:"additional_environment_variables,omitempty"` } -func (i *HostInstaller) writeYamlConfig(path string, yml interface{}, perm os.FileMode, agentOwner bool) error { - data, err := yaml.Marshal(yml) - if err != nil { - return err +// InjectTracerConfigEnvVar represents an environment variable to inject +type InjectTracerConfigEnvVar struct { + Key string `yaml:"key"` + Value string `yaml:"value"` +} + +// mergeConfig merges the current config with the setup config. +// +// The values are merged as follows: +// - Scalars: the override value is used +// - Lists: the override list is used +// - Maps: the override map is recursively merged into the base map +func mergeConfig(base interface{}, override interface{}) (interface{}, error) { + if base == nil { + return override, nil } - dir := filepath.Dir(path) - if err := os.MkdirAll(dir, 0755); err != nil { - return fmt.Errorf("failed to create directory %s: %v", dir, err) + if override == nil { + // this allows to override a value with nil + return nil, nil } - if err = os.WriteFile(path, data, perm); err != nil { - return fmt.Errorf("failed to write to %s: %v", path, err) + if isScalar(base) && isScalar(override) { + return override, nil } - // Change ownership of the file to the agent user - // ddUID=0 happens in local test environments - if agentOwner && i.ddUID != 0 { - if err := os.Chown(path, i.ddUID, i.ddGID); err != nil { - return fmt.Errorf("failed to change ownership of %s: %v", path, err) - } + if isList(base) && isList(override) { + return override, nil } - return nil -} - -func convertTagsToYaml(tags []tag) []interface{} { - result := make([]interface{}, 0, len(tags)) - for _, tag := range tags { - result = append(result, fmt.Sprintf("%s:%s", tag.key, tag.value)) + if isMap(base) && isMap(override) { + return mergeMap(base.(map[string]interface{}), override.(map[string]interface{})) } - return result + return nil, fmt.Errorf("could not merge %T with %T", base, override) } -func (i *HostInstaller) writeConfigs() error { - if len(i.hostTags) > 0 { - i.AddAgentConfig("tags", convertTagsToYaml(i.hostTags)) - } - - if err := i.writeYamlConfig(datadogConfFile, i.agentConfig, 0640, true); err != nil { - return err - } - if len(i.logsConfig.Logs) > 0 { - if err := i.writeYamlConfig(logsConfFile, i.logsConfig, 0644, true); err != nil { - return err - } +func mergeMap(base, override map[string]interface{}) (map[string]interface{}, error) { + merged := make(map[string]interface{}) + for k, v := range base { + merged[k] = v } - if len(i.sparkConfig.Instances) > 0 { - if err := i.writeYamlConfig(sparkConfigFile, i.sparkConfig, 0644, true); err != nil { - return err + for k := range override { + v, err := mergeConfig(base[k], override[k]) + if err != nil { + return nil, fmt.Errorf("could not merge key %v: %w", k, err) } + merged[k] = v } - if len(i.injectorConfig.EnvsToInject) > 0 { - if err := i.writeYamlConfig(injectTracerConfigFile, i.injectorConfig, 0644, false); err != nil { - return err - } - } - return nil + return merged, nil } -// ConfigureAndInstall writes configurations to disk and installs desired packages -func (i *HostInstaller) ConfigureAndInstall(ctx context.Context) error { - if err := i.writeConfigs(); err != nil { - return fmt.Errorf("failed to write configurations: %w", err) - } +func isList(i interface{}) bool { + _, ok := i.([]interface{}) + return ok +} - exePath, err := os.Executable() - if err != nil { - return fmt.Errorf("failed to get executable path: %w", err) - } - cmd := exec.NewInstallerExec(i.env, exePath) +func isMap(i interface{}) bool { + _, ok := i.(map[string]interface{}) + return ok +} - if i.injectorVersion != "" { - if err := cmd.Install(ctx, oci.PackageURL(i.env, "datadog-apm-inject", i.injectorVersion), nil); err != nil { - return fmt.Errorf("failed to install injector: %w", err) - } - } - if i.javaVersion != "" { - if err := cmd.Install(ctx, oci.PackageURL(i.env, "datadog-apm-library-java", i.javaVersion), nil); err != nil { - return fmt.Errorf("failed to install java library: %w", err) - } - } - if i.agentVersion != "" { - if err := cmd.Install(ctx, oci.PackageURL(i.env, "datadog-agent", i.agentVersion), nil); err != nil { - return fmt.Errorf("failed to install Databricks agent: %w", err) - } - } - return nil +func isScalar(i interface{}) bool { + return !isList(i) && !isMap(i) } diff --git a/pkg/fleet/installer/setup/common/config_test.go b/pkg/fleet/installer/setup/common/config_test.go index bfe2809f39087..7c02cd3999e02 100644 --- a/pkg/fleet/installer/setup/common/config_test.go +++ b/pkg/fleet/installer/setup/common/config_test.go @@ -5,7 +5,6 @@ //go:build !windows -// Package common contains the HostInstaller struct which is used to write the agent agentConfiguration to disk package common import ( @@ -14,48 +13,155 @@ import ( "testing" "github.com/stretchr/testify/assert" - - "github.com/DataDog/datadog-agent/pkg/fleet/installer/env" + "gopkg.in/yaml.v3" ) -func assertFileContent(t *testing.T, file, content string) { - b, err := os.ReadFile(file) +func TestEmptyConfig(t *testing.T) { + tempDir := t.TempDir() + config := Config{} + config.DatadogYAML.APIKey = "1234567890" // Required field + + err := writeConfigs(config, tempDir) + assert.NoError(t, err) + + // Check datadog.yaml + datadogConfigPath := filepath.Join(tempDir, datadogConfFile) + info, err := os.Stat(datadogConfigPath) + assert.NoError(t, err) + assert.Equal(t, os.FileMode(0640), info.Mode()) + datadogYAML, err := os.ReadFile(datadogConfigPath) + assert.NoError(t, err) + var datadog map[string]interface{} + err = yaml.Unmarshal(datadogYAML, &datadog) + assert.NoError(t, err) + assert.Equal(t, map[string]interface{}{"api_key": "1234567890"}, datadog) + + // Assert no other files are created + dir, err := os.ReadDir(tempDir) + assert.NoError(t, err) + assert.Len(t, dir, 1) +} + +func TestMergeConfig(t *testing.T) { + tempDir := t.TempDir() + oldConfig := `--- +api_key: "0987654321" +hostname: "old_hostname" +env: "old_env" +` + err := os.WriteFile(filepath.Join(tempDir, datadogConfFile), []byte(oldConfig), 0644) + assert.NoError(t, err) + config := Config{} + config.DatadogYAML.APIKey = "1234567890" // Required field + config.DatadogYAML.Hostname = "new_hostname" + config.DatadogYAML.LogsEnabled = true + + err = writeConfigs(config, tempDir) + assert.NoError(t, err) + + // Check datadog.yaml + datadogConfigPath := filepath.Join(tempDir, datadogConfFile) + datadogYAML, err := os.ReadFile(datadogConfigPath) + assert.NoError(t, err) + var datadog map[string]interface{} + err = yaml.Unmarshal(datadogYAML, &datadog) + assert.NoError(t, err) + assert.Equal(t, map[string]interface{}{ + "api_key": "1234567890", + "hostname": "new_hostname", + "env": "old_env", + "logs_enabled": true, + }, datadog) +} + +func TestInjectTracerConfig(t *testing.T) { + tempDir := t.TempDir() + config := Config{} + config.InjectTracerYAML = InjectTracerConfig{ + Version: 1, + ConfigSources: "env", + AdditionalEnvironmentVariables: []InjectTracerConfigEnvVar{ + { + Key: "DD_ENV", + Value: "prod", + }, + }, + } + + err := writeConfigs(config, tempDir) + assert.NoError(t, err) + + // Check inject/tracer.yaml + injectTracerConfigPath := filepath.Join(tempDir, injectTracerConfigFile) + assert.FileExists(t, injectTracerConfigPath) + injectTracerYAML, err := os.ReadFile(injectTracerConfigPath) + assert.NoError(t, err) + var injectTracer map[string]interface{} + err = yaml.Unmarshal(injectTracerYAML, &injectTracer) assert.NoError(t, err) - assert.Equal(t, content, string(b)) + assert.Equal(t, map[string]interface{}{ + "version": 1, + "config_sources": "env", + "additional_environment_variables": []interface{}{ + map[string]interface{}{ + "key": "DD_ENV", + "value": "prod", + }, + }, + }, injectTracer) } -func TestAgentConfigs(t *testing.T) { - configDir = t.TempDir() - datadogConfFile = filepath.Join(configDir, "datadog.yaml") - logsConfFile = filepath.Join(configDir, "conf.d/configured_at_install_logs.yaml") - sparkConfigFile = filepath.Join(configDir, "conf.d/spark.d/spark.yaml") - - i, err := newHostInstaller(&env.Env{APIKey: "a"}, 0, 0) - assert.NotNil(t, i) - assert.Nil(t, err) - - i.AddAgentConfig("key", "value") - i.AddLogConfig(LogConfig{Type: "file", Path: "/var/log/app.log", Service: "app"}) - i.AddHostTag("k1", "v1") - i.AddHostTag("k2", "v2") - i.AddSparkInstance(SparkInstance{ClusterName: "cluster", SparkURL: "http://localhost:8080"}) - - assert.NoError(t, i.writeConfigs()) - assertFileContent(t, datadogConfFile, `api_key: a -key: value -logs_enabled: true -tags: -- k1:v1 -- k2:v2 -`) - - assertFileContent(t, logsConfFile, `logs: -- type: file - path: /var/log/app.log - service: app -`) - assertFileContent(t, sparkConfigFile, `instances: -- spark_url: http://localhost:8080 - cluster_name: cluster -`) +func TestIntegrationConfigInstanceSpark(t *testing.T) { + tempDir := t.TempDir() + config := Config{ + IntegrationConfigs: make(map[string]IntegrationConfig), + } + config.IntegrationConfigs["spark.d/kebabricks.yaml"] = IntegrationConfig{ + Logs: []IntegrationConfigLogs{ + { + Type: "file", + Path: "/databricks/spark/work/*/*/stderr", + Source: "worker_stderr", + Service: "databricks", + }, + }, + Instances: []any{ + IntegrationConfigInstanceSpark{ + SparkURL: "http://localhost:4040", + SparkClusterMode: "spark_driver_mode", + ClusterName: "big-kebab-data", + StreamingMetrics: true, + }, + }, + } + + err := writeConfigs(config, tempDir) + assert.NoError(t, err) + + // Check spark.d/kebabricks.yaml + sparkConfigPath := filepath.Join(tempDir, "conf.d", "spark.d", "kebabricks.yaml") + assert.FileExists(t, sparkConfigPath) + sparkYAML, err := os.ReadFile(sparkConfigPath) + assert.NoError(t, err) + var spark map[string]interface{} + err = yaml.Unmarshal(sparkYAML, &spark) + assert.NoError(t, err) + assert.Equal(t, map[string]interface{}{ + "logs": []interface{}{ + map[string]interface{}{ + "type": "file", + "path": "/databricks/spark/work/*/*/stderr", + "service": "databricks", + "source": "worker_stderr", + }, + }, + "instances": []interface{}{ + map[string]interface{}{ + "spark_url": "http://localhost:4040", + "spark_cluster_mode": "spark_driver_mode", + "cluster_name": "big-kebab-data", + "streaming_metrics": true, + }, + }, + }, spark) } diff --git a/pkg/fleet/installer/setup/common/packages.go b/pkg/fleet/installer/setup/common/packages.go new file mode 100644 index 0000000000000..c4c0c091ea048 --- /dev/null +++ b/pkg/fleet/installer/setup/common/packages.go @@ -0,0 +1,74 @@ +// Unless explicitly stated otherwise all files in this repository are licensed +// under the Apache License Version 2.0. +// This product includes software developed at Datadog (https://www.datadoghq.com/). +// Copyright 2016-present Datadog, Inc. + +package common + +import "fmt" + +const ( + // DatadogInstallerPackage is the datadog installer package + DatadogInstallerPackage string = "datadog-installer" + // DatadogAgentPackage is the datadog agent package + DatadogAgentPackage string = "datadog-agent" + // DatadogAPMInjectPackage is the datadog apm inject package + DatadogAPMInjectPackage string = "datadog-apm-inject" + // DatadogAPMLibraryJavaPackage is the datadog apm library java package + DatadogAPMLibraryJavaPackage string = "datadog-apm-library-java" + // DatadogAPMLibraryPythonPackage is the datadog apm library python package + DatadogAPMLibraryPythonPackage string = "datadog-apm-library-python" + // DatadogAPMLibraryRubyPackage is the datadog apm library ruby package + DatadogAPMLibraryRubyPackage string = "datadog-apm-library-ruby" + // DatadogAPMLibraryJSPackage is the datadog apm library js package + DatadogAPMLibraryJSPackage string = "datadog-apm-library-js" + // DatadogAPMLibraryDotNetPackage is the datadog apm library dotnet package + DatadogAPMLibraryDotNetPackage string = "datadog-apm-library-dotnet" + // DatadogAPMLibraryPHPPackage is the datadog apm library php package + DatadogAPMLibraryPHPPackage string = "datadog-apm-library-php" +) + +var ( + order = []string{ + DatadogInstallerPackage, + DatadogAgentPackage, + DatadogAPMInjectPackage, + DatadogAPMLibraryJavaPackage, + DatadogAPMLibraryPythonPackage, + DatadogAPMLibraryRubyPackage, + DatadogAPMLibraryJSPackage, + DatadogAPMLibraryDotNetPackage, + DatadogAPMLibraryPHPPackage, + } +) + +func resolvePackages(packages Packages) []packageWithVersion { + var resolved []packageWithVersion + for _, pkg := range order { + if p, ok := packages.install[pkg]; ok { + resolved = append(resolved, p) + } + } + if len(resolved) != len(packages.install) { + panic(fmt.Sprintf("unknown package requested: %v", packages.install)) + } + return resolved +} + +// Packages is a list of packages to install +type Packages struct { + install map[string]packageWithVersion +} + +type packageWithVersion struct { + name string + version string +} + +// Install marks a package to be installed +func (p *Packages) Install(pkg string, version string) { + p.install[pkg] = packageWithVersion{ + name: pkg, + version: version, + } +} diff --git a/pkg/fleet/installer/setup/common/setup.go b/pkg/fleet/installer/setup/common/setup.go new file mode 100644 index 0000000000000..dbd48a7f83028 --- /dev/null +++ b/pkg/fleet/installer/setup/common/setup.go @@ -0,0 +1,95 @@ +// Unless explicitly stated otherwise all files in this repository are licensed +// under the Apache License Version 2.0. +// This product includes software developed at Datadog (https://www.datadoghq.com/). +// Copyright 2016-present Datadog, Inc. + +// Package common defines the Setup structure that allows setup scripts to define packages and configurations to install. +package common + +import ( + "context" + "errors" + "fmt" + "os" + + "github.com/DataDog/datadog-agent/pkg/fleet/installer" + "github.com/DataDog/datadog-agent/pkg/fleet/installer/env" + "github.com/DataDog/datadog-agent/pkg/fleet/installer/oci" + "gopkg.in/DataDog/dd-trace-go.v1/ddtrace" + "gopkg.in/DataDog/dd-trace-go.v1/ddtrace/tracer" +) + +const ( + installerOCILayoutURL = "file://." // the installer OCI layout is written by the downloader in the current directory +) + +var ( + // ErrNoAPIKey is returned when no API key is provided. + ErrNoAPIKey = errors.New("no API key provided") +) + +// Setup allows setup scripts to define packages and configurations to install. +type Setup struct { + configDir string + installer installer.Installer + + Env *env.Env + Ctx context.Context + Span ddtrace.Span + Packages Packages + Config Config +} + +// NewSetup creates a new Setup structure with some default values. +func NewSetup(ctx context.Context, env *env.Env, name string) (*Setup, error) { + if env.APIKey == "" { + return nil, ErrNoAPIKey + } + installer, err := installer.NewInstaller(env) + if err != nil { + return nil, fmt.Errorf("failed to create installer: %w", err) + } + span, ctx := tracer.StartSpanFromContext(ctx, fmt.Sprintf("setup.%s", name)) + s := &Setup{ + configDir: configDir, + installer: installer, + Env: env, + Ctx: ctx, + Span: span, + Config: Config{ + DatadogYAML: DatadogConfig{ + APIKey: env.APIKey, + Hostname: os.Getenv("DD_HOSTNAME"), + Site: env.Site, + Env: os.Getenv("DD_ENV"), + }, + IntegrationConfigs: make(map[string]IntegrationConfig), + }, + Packages: Packages{ + install: make(map[string]packageWithVersion), + }, + } + return s, nil +} + +// Run installs the packages and writes the configurations +func (s *Setup) Run() (err error) { + defer func() { s.Span.Finish(tracer.WithError(err)) }() + err = writeConfigs(s.Config, s.configDir) + if err != nil { + return fmt.Errorf("failed to write configuration: %w", err) + } + err = s.installer.Install(s.Ctx, installerOCILayoutURL, nil) + if err != nil { + return fmt.Errorf("failed to install installer: %w", err) + } + packages := resolvePackages(s.Packages) + for _, p := range packages { + url := oci.PackageURL(s.Env, p.name, p.version) + err = s.installer.Install(s.Ctx, url, nil) + if err != nil { + return fmt.Errorf("failed to install package %s: %w", url, err) + } + } + return nil +} diff --git a/pkg/fleet/installer/setup/djm/databricks.go b/pkg/fleet/installer/setup/djm/databricks.go index 21db9b4b7a934..2dc8909c58e8b 100644 --- a/pkg/fleet/installer/setup/djm/databricks.go +++ b/pkg/fleet/installer/setup/djm/databricks.go @@ -3,188 +3,149 @@ // This product includes software developed at Datadog (https://www.datadoghq.com/). // Copyright 2016-present Datadog, Inc. -//go:build !windows - // Package djm contains data-jobs-monitoring installation logic package djm import ( - "context" + "fmt" "os" - "github.com/DataDog/datadog-agent/pkg/fleet/installer/env" "github.com/DataDog/datadog-agent/pkg/fleet/installer/setup/common" "github.com/DataDog/datadog-agent/pkg/util/log" - "gopkg.in/DataDog/dd-trace-go.v1/ddtrace/tracer" ) const ( databricksInjectorVersion = "0.21.0-1" databricksJavaVersion = "1.41.1-1" databricksAgentVersion = "7.57.2-1" - logsService = "databricks" ) -type databricksSetup struct { - ctx context.Context - *common.HostInstaller - setupIssues []string -} +var ( + envToTags = map[string]string{ + "DATABRICKS_WORKSPACE": "workspace", + "DB_CLUSTER_NAME": "databricks_cluster_name", + "DB_CLUSTER_ID": "databricks_cluster_id", + "DB_NODE_TYPE": "databricks_node_type", + } + driverLogs = []common.IntegrationConfigLogs{ + { + Type: "file", + Path: "/databricks/driver/logs/*.log", + Source: "driver_logs", + Service: "databricks", + }, + { + Type: "file", + Path: "/databricks/driver/logs/stderr", + Source: "driver_stderr", + Service: "databricks", + }, + { + Type: "file", + Path: "/databricks/driver/logs/stdout", + Source: "driver_stdout", + Service: "databricks", + }, + } + workerLogs = []common.IntegrationConfigLogs{ + { + Type: "file", + Path: "/databricks/spark/work/*/*/*.log", + Source: "worker_logs", + Service: "databricks", + }, + { + Type: "file", + Path: "/databricks/spark/work/*/*/stderr", + Source: "worker_stderr", + Service: "databricks", + }, + { + Type: "file", + Path: "/databricks/spark/work/*/*/stdout", + Source: "worker_stdout", + Service: "databricks", + }, + } + tracerEnvConfig = []common.InjectTracerConfigEnvVar{ + { + Key: "DD_DATA_JOBS_ENABLED", + Value: "true", + }, + { + Key: "DD_INTEGRATIONS_ENABLED", + Value: "false", + }, + } +) // SetupDatabricks sets up the Databricks environment -func SetupDatabricks(ctx context.Context, env *env.Env) (err error) { - span, ctx := tracer.StartSpanFromContext(ctx, "setup.databricks") - defer func() { span.Finish(tracer.WithError(err)) }() +func SetupDatabricks(s *common.Setup) error { + s.Packages.Install(common.DatadogAgentPackage, databricksAgentVersion) - i, err := common.NewHostInstaller(env) + hostname, err := os.Hostname() if err != nil { - return err + return fmt.Errorf("failed to get hostname: %w", err) } - ds := &databricksSetup{ - ctx: ctx, - HostInstaller: i, + s.Config.DatadogYAML.Hostname = hostname + s.Config.DatadogYAML.DJM.Enabled = true + s.Config.DatadogYAML.ExpectedTagsDuration = "10m" + s.Config.DatadogYAML.ProcessConfig.ExpvarPort = 6063 // avoid port conflict on 6062 + for env, tag := range envToTags { + if val, ok := os.LookupEnv(env); ok { + s.Config.DatadogYAML.Tags = append(s.Config.DatadogYAML.Tags, tag+":"+val) + } } - return ds.setup() -} - -func (ds *databricksSetup) setup() error { - // agent binary to install - ds.SetAgentVersion(databricksAgentVersion) - - // avoid port conflict - ds.AddAgentConfig("process_config.expvar_port", -1) - ds.AddAgentConfig("expected_tags_duration", "10m") - ds.AddAgentConfig("djm_config.enabled", true) - - ds.extractHostTagsFromEnv() - span, _ := tracer.SpanFromContext(ds.ctx) switch os.Getenv("DB_IS_DRIVER") { case "TRUE": - span.SetTag("spark_node", "driver") - return ds.setupDatabricksDriver() + setupDatabricksDriver(s) default: - span.SetTag("spark_node", "worker") - return ds.setupDatabricksExecutor() + setupDatabricksWorker(s) } + return nil } -type varExtraction struct { - envVar string - tagKey string -} +func setupDatabricksDriver(s *common.Setup) { + s.Span.SetTag("spark_node", "driver") -var varExtractions = []varExtraction{ - {"DATABRICKS_WORKSPACE", "workspace"}, - {"DB_CLUSTER_NAME", "databricks_cluster_name"}, - {"DB_CLUSTER_ID", "databricks_cluster_id"}, - {"DB_NODE_TYPE", "databricks_node_type"}, -} + s.Packages.Install(common.DatadogAPMInjectPackage, databricksInjectorVersion) + s.Packages.Install(common.DatadogAPMLibraryJavaPackage, databricksJavaVersion) + + s.Config.DatadogYAML.Tags = append(s.Config.DatadogYAML.Tags, "node_type:driver") + s.Config.InjectTracerYAML.AdditionalEnvironmentVariables = tracerEnvConfig -func (ds *databricksSetup) extractHostTagsFromEnv() { - for _, ve := range varExtractions { - if val, ok := os.LookupEnv(ve.envVar); ok { - ds.AddHostTag(ve.tagKey, val) - continue + var sparkIntegration common.IntegrationConfig + if os.Getenv("DRIVER_LOGS_ENABLED") == "true" { + s.Config.DatadogYAML.LogsEnabled = true + sparkIntegration.Logs = driverLogs + } + if os.Getenv("DB_DRIVER_IP") != "" || os.Getenv("DB_DRIVER_PORT") != "" { + sparkIntegration.Instances = []any{ + common.IntegrationConfigInstanceSpark{ + SparkURL: "http://" + os.Getenv("DB_DRIVER_IP") + ":" + os.Getenv("DB_DRIVER_PORT"), + SparkClusterMode: "spark_driver_mode", + ClusterName: os.Getenv("DB_CLUSTER_NAME"), + StreamingMetrics: true, + }, } - ds.setupIssues = append(ds.setupIssues, ve.envVar+"_not_set") + } else { + log.Warn("DB_DRIVER_IP or DB_DRIVER_PORT not set") } + s.Config.IntegrationConfigs["spark.d/databricks.yaml"] = sparkIntegration } -func (ds *databricksSetup) setupDatabricksDriver() error { - ds.AddHostTag("node_type", "driver") - - ds.driverLogCollection() - - ds.setupAgentSparkCheck() - - ds.AddTracerEnv("DD_DATA_JOBS_ENABLED", "true") - ds.AddTracerEnv("DD_INTEGRATIONS_ENABLED", "false") - - // APM binaries to install - ds.SetInjectorVersion(databricksInjectorVersion) - ds.SetJavaTracerVersion(databricksJavaVersion) - - return ds.ConfigureAndInstall(ds.ctx) -} - -func (ds *databricksSetup) setupDatabricksExecutor() error { - ds.AddHostTag("node_type", "worker") - ds.workerLogCollection() - return ds.ConfigureAndInstall(ds.ctx) -} +func setupDatabricksWorker(s *common.Setup) { + s.Span.SetTag("spark_node", "worker") -func (ds *databricksSetup) setupAgentSparkCheck() { - driverIP := os.Getenv("DB_DRIVER_IP") - if driverIP == "" { - log.Warn("DB_DRIVER_IP not set") - return - } - driverPort := os.Getenv("DB_DRIVER_PORT") - if driverPort == "" { - log.Warn("DB_DRIVER_PORT not set") - return - } - clusterName := os.Getenv("DB_CLUSTER_NAME") - - ds.AddSparkInstance(common.SparkInstance{ - SparkURL: "http://" + driverIP + ":" + driverPort, - SparkClusterMode: "spark_driver_mode", - ClusterName: clusterName, - StreamingMetrics: true, - }) -} + s.Packages.Install(common.DatadogAgentPackage, databricksAgentVersion) -func (ds *databricksSetup) driverLogCollection() { - if os.Getenv("DRIVER_LOGS_ENABLED") != "true" { - return - } - span, _ := tracer.SpanFromContext(ds.ctx) - span.SetTag("driver_logs", "enabled") - log.Info("Enabling logs collection on the driver") - ds.AddLogConfig(common.LogConfig{ - Type: "file", - Path: "/databricks/driver/logs/*.log", - Source: "driver_logs", - Service: logsService, - }) - ds.AddLogConfig(common.LogConfig{ - Type: "file", - Path: "/databricks/driver/logs/stderr", - Source: "driver_stderr", - Service: logsService, - }) - ds.AddLogConfig(common.LogConfig{ - Type: "file", - Path: "/databricks/driver/logs/stdout", - Source: "driver_stdout", - Service: logsService, - }) -} + s.Config.DatadogYAML.Tags = append(s.Config.DatadogYAML.Tags, "node_type:worker") -func (ds *databricksSetup) workerLogCollection() { - if os.Getenv("WORKER_LOGS_ENABLED") != "true" { - return + var sparkIntegration common.IntegrationConfig + if os.Getenv("WORKER_LOGS_ENABLED") == "true" { + s.Config.DatadogYAML.LogsEnabled = true + sparkIntegration.Logs = workerLogs } - span, _ := tracer.SpanFromContext(ds.ctx) - span.SetTag("worker_logs", "enabled") - log.Info("Enabling logs collection on the executor") - ds.AddLogConfig(common.LogConfig{ - Type: "file", - Path: "/databricks/spark/work/*/*/*.log", - Source: "worker_logs", - Service: logsService, - }) - ds.AddLogConfig(common.LogConfig{ - Type: "file", - Path: "/databricks/spark/work/*/*/stderr", - Source: "worker_stderr", - Service: logsService, - }) - ds.AddLogConfig(common.LogConfig{ - Type: "file", - Path: "/databricks/spark/work/*/*/stdout", - Source: "worker_stdout", - Service: logsService, - }) + s.Config.IntegrationConfigs["spark.d/databricks.yaml"] = sparkIntegration } diff --git a/pkg/fleet/installer/setup/djm/databricks_windows.go b/pkg/fleet/installer/setup/djm/databricks_windows.go deleted file mode 100644 index 2bad2770e3467..0000000000000 --- a/pkg/fleet/installer/setup/djm/databricks_windows.go +++ /dev/null @@ -1,21 +0,0 @@ -// Unless explicitly stated otherwise all files in this repository are licensed -// under the Apache License Version 2.0. -// This product includes software developed at Datadog (https://www.datadoghq.com/). -// Copyright 2016-present Datadog, Inc. - -//go:build windows - -// Package djm contains data-jobs-monitoring installation logic -package djm - -import ( - "context" - "errors" - - "github.com/DataDog/datadog-agent/pkg/fleet/installer/env" -) - -// SetupDatabricks is a not supported on windows -func SetupDatabricks(_ context.Context, _ *env.Env) error { - return errors.New("djm is not supported on windows") -} diff --git a/pkg/fleet/installer/setup/setup.go b/pkg/fleet/installer/setup/setup.go index 0daa479d73b6e..3b01d320c9975 100644 --- a/pkg/fleet/installer/setup/setup.go +++ b/pkg/fleet/installer/setup/setup.go @@ -11,7 +11,7 @@ import ( "fmt" "github.com/DataDog/datadog-agent/pkg/fleet/installer/env" - "github.com/DataDog/datadog-agent/pkg/fleet/installer/packages" + "github.com/DataDog/datadog-agent/pkg/fleet/installer/setup/common" "github.com/DataDog/datadog-agent/pkg/fleet/installer/setup/djm" "github.com/DataDog/datadog-agent/pkg/fleet/internal/exec" "github.com/DataDog/datadog-agent/pkg/fleet/internal/paths" @@ -24,18 +24,20 @@ const ( // Setup installs Datadog. func Setup(ctx context.Context, env *env.Env, flavor string) error { + s, err := common.NewSetup(ctx, env, flavor) + if err != nil { + return err + } switch flavor { case FlavorDatabricks: - if err := packages.SetupInstaller(ctx); err != nil { - return fmt.Errorf("failed to setup installer: %w", err) - } - if err := djm.SetupDatabricks(ctx, env); err != nil { - return fmt.Errorf("failed to setup Databricks: %w", err) - } - return nil + err = djm.SetupDatabricks(s) default: return fmt.Errorf("unknown setup flavor %s", flavor) } + if err != nil { + return err + } + return s.Run() } // Agent7InstallScript is the setup used by the agent7 install script.