diff --git a/cmd/installer/subcommands/installer/command.go b/cmd/installer/subcommands/installer/command.go index 2a612cfe1750c..bb380cbfd79ab 100644 --- a/cmd/installer/subcommands/installer/command.go +++ b/cmd/installer/subcommands/installer/command.go @@ -293,7 +293,7 @@ func installCommand() *cobra.Command { if err != nil { return err } - defer i.stop(err) + defer func() { i.stop(err) }() i.span.SetTag("params.url", args[0]) return i.Install(i.ctx, args[0], installArgs) }, @@ -313,7 +313,7 @@ func removeCommand() *cobra.Command { if err != nil { return err } - defer i.stop(err) + defer func() { i.stop(err) }() i.span.SetTag("params.package", args[0]) return i.Remove(i.ctx, args[0]) }, @@ -332,7 +332,7 @@ func purgeCommand() *cobra.Command { if err != nil { return err } - defer i.stop(err) + defer func() { i.stop(err) }() i.Purge(i.ctx) return nil }, @@ -351,7 +351,7 @@ func installExperimentCommand() *cobra.Command { if err != nil { return err } - defer i.stop(err) + defer func() { i.stop(err) }() i.span.SetTag("params.url", args[0]) return i.InstallExperiment(i.ctx, args[0]) }, @@ -370,7 +370,7 @@ func removeExperimentCommand() *cobra.Command { if err != nil { return err } - defer i.stop(err) + defer func() { i.stop(err) }() i.span.SetTag("params.package", args[0]) return i.RemoveExperiment(i.ctx, args[0]) }, @@ -389,7 +389,7 @@ func promoteExperimentCommand() *cobra.Command { if err != nil { return err } - defer i.stop(err) + defer func() { i.stop(err) }() i.span.SetTag("params.package", args[0]) return i.PromoteExperiment(i.ctx, args[0]) }, @@ -408,7 +408,7 @@ func installConfigExperimentCommand() *cobra.Command { if err != nil { return err } - defer func() { i.Stop(err) }() + defer func() { i.stop(err) }() i.span.SetTag("params.package", args[0]) i.span.SetTag("params.version", args[1]) return i.InstallConfigExperiment(i.ctx, args[0], args[1]) @@ -428,7 +428,7 @@ func removeConfigExperimentCommand() *cobra.Command { if err != nil { return err } - defer func() { i.Stop(err) }() + defer func() { i.stop(err) }() i.span.SetTag("params.package", args[0]) return i.RemoveConfigExperiment(i.ctx, args[0]) }, @@ -447,7 +447,7 @@ func promoteConfigExperimentCommand() *cobra.Command { if err != nil { return err } - defer func() { i.Stop(err) }() + defer func() { i.stop(err) }() i.span.SetTag("params.package", args[0]) return i.PromoteConfigExperiment(i.ctx, args[0]) }, @@ -466,7 +466,7 @@ func garbageCollectCommand() *cobra.Command { if err != nil { return err } - defer i.stop(err) + defer func() { i.stop(err) }() return i.GarbageCollect(i.ctx) }, } @@ -489,7 +489,7 @@ func isInstalledCommand() *cobra.Command { if err != nil { return err } - defer i.stop(err) + defer func() { i.stop(err) }() installed, err := i.IsInstalled(i.ctx, args[0]) if err != nil { return err diff --git a/pkg/config/remote/service/service.go b/pkg/config/remote/service/service.go index f09c59df87d88..8994a60933f26 100644 --- a/pkg/config/remote/service/service.go +++ b/pkg/config/remote/service/service.go @@ -17,14 +17,15 @@ import ( "errors" "expvar" "fmt" - "github.com/DataDog/datadog-agent/pkg/remoteconfig/state" - "gopkg.in/DataDog/dd-trace-go.v1/ddtrace/tracer" "net/url" "path" "strconv" "sync" "time" + "github.com/DataDog/datadog-agent/pkg/remoteconfig/state" + "gopkg.in/DataDog/dd-trace-go.v1/ddtrace/tracer" + "github.com/DataDog/go-tuf/data" tufutil "github.com/DataDog/go-tuf/util" "github.com/benbjohnson/clock" @@ -247,6 +248,7 @@ type options struct { apiKey string traceAgentEnv string databaseFileName string + databaseFilePath string configRootOverride string directorRootOverride string clientCacheBypassLimit int @@ -261,6 +263,7 @@ var defaultOptions = options{ apiKey: "", traceAgentEnv: "", databaseFileName: "remote-config.db", + databaseFilePath: "", configRootOverride: "", directorRootOverride: "", clientCacheBypassLimit: defaultCacheBypassLimit, @@ -283,6 +286,11 @@ func WithDatabaseFileName(fileName string) func(s *options) { return func(s *options) { s.databaseFileName = fileName } } +// WithDatabasePath sets the service database path +func WithDatabasePath(path string) func(s *options) { + return func(s *options) { s.databaseFilePath = path } +} + // WithConfigRootOverride sets the service config root override func WithConfigRootOverride(site string, override string) func(s *options) { return func(opts *options) { @@ -410,7 +418,11 @@ func NewService(cfg model.Reader, rcType, baseRawURL, hostname string, tagsGette return nil, err } - dbPath := path.Join(cfg.GetString("run_path"), options.databaseFileName) + databaseFilePath := cfg.GetString("run_path") + if options.databaseFilePath != "" { + databaseFilePath = options.databaseFilePath + } + dbPath := path.Join(databaseFilePath, options.databaseFileName) db, err := openCacheDB(dbPath, agentVersion, authKeys.apiKey) if err != nil { return nil, err diff --git a/pkg/fleet/daemon/daemon.go b/pkg/fleet/daemon/daemon.go index 11c5d7e0d76d8..dab1305100748 100644 --- a/pkg/fleet/daemon/daemon.go +++ b/pkg/fleet/daemon/daemon.go @@ -41,7 +41,7 @@ const ( // gcInterval is the interval at which the GC will run gcInterval = 1 * time.Hour // refreshStateInterval is the interval at which the state will be refreshed - refreshStateInterval = 1 * time.Minute + refreshStateInterval = 30 * time.Second ) // Daemon is the fleet daemon in charge of remote install, updates and configuration. @@ -562,7 +562,7 @@ func (d *daemonImpl) resolveRemoteConfigVersion(ctx context.Context, pkg string) } config, err := d.cdn.Get(ctx, pkg) if err != nil { - return "", fmt.Errorf("could not get cdn config: %w", err) + return "", err } return config.Version(), nil } @@ -602,9 +602,6 @@ func (d *daemonImpl) refreshState(ctx context.Context) { } configVersion, err := d.resolveRemoteConfigVersion(ctx, pkg) - if err != nil { - log.Errorf("could not get agent remote config version: %v", err) - } if err == nil { p.RemoteConfigVersion = configVersion } else if err != cdn.ErrProductNotSupported { diff --git a/pkg/fleet/env/env.go b/pkg/fleet/env/env.go index 18cf17ea6ed06..dbdce9c709b68 100644 --- a/pkg/fleet/env/env.go +++ b/pkg/fleet/env/env.go @@ -32,6 +32,7 @@ const ( envAgentMinorVersion = "DD_AGENT_MINOR_VERSION" envApmLanguages = "DD_APM_INSTRUMENTATION_LANGUAGES" envCDNLocalDirPath = "DD_INSTALLER_DEBUG_CDN_LOCAL_DIR_PATH" + envCDNEnabled = "DD_INSTALLER_CDN_ENABLED" ) var defaultEnv = Env{ @@ -88,6 +89,7 @@ type Env struct { InstallScript InstallScriptEnv + CDNEnabled bool CDNLocalDirPath string } @@ -118,6 +120,7 @@ func FromEnv() *Env { InstallScript: installScriptEnvFromEnv(), + CDNEnabled: strings.ToLower(os.Getenv(envCDNEnabled)) == "true", CDNLocalDirPath: getEnvOrDefault(envCDNLocalDirPath, ""), } } diff --git a/pkg/fleet/internal/cdn/cdn.go b/pkg/fleet/internal/cdn/cdn.go index 8bbaa37d02c15..7891c7b697904 100644 --- a/pkg/fleet/internal/cdn/cdn.go +++ b/pkg/fleet/internal/cdn/cdn.go @@ -10,6 +10,7 @@ import ( "context" "errors" "regexp" + "runtime" "github.com/DataDog/datadog-agent/pkg/fleet/env" ) @@ -38,10 +39,36 @@ type CDN interface { Close() error } -// New creates a new CDN. +// New creates a new CDN and chooses the implementation depending +// on the environment func New(env *env.Env, configDBPath string) (CDN, error) { + if runtime.GOOS == "windows" { + // There's an assumption on windows that some directories are already there + // but they are in fact created by the regular CDN implementation. Until + // there is a fix on windows we keep the previous CDN behaviour for them + return newCDNHTTP(env, configDBPath) + } + + if !env.RemotePolicies { + // Remote policies are not enabled -- we don't need the CDN + // and we don't want to create the directories that the CDN + // implementation would create. We return a no-op CDN to avoid + // nil pointer dereference. + return newCDNNoop() + } + if env.CDNLocalDirPath != "" { - return newLocal(env) + // Mock the CDN for local development or testing + return newCDNLocal(env) } - return newRemote(env, configDBPath) + + if !env.CDNEnabled { + // Remote policies are enabled but we don't want to use the CDN + // as it's still in development. We use standard remote config calls + // instead (dubbed "direct" CDN). + return newCDNRC(env, configDBPath) + } + + // Regular CDN with the cloudfront distribution + return newCDNHTTP(env, configDBPath) } diff --git a/pkg/fleet/internal/cdn/cdn_remote.go b/pkg/fleet/internal/cdn/cdn_http.go similarity index 89% rename from pkg/fleet/internal/cdn/cdn_remote.go rename to pkg/fleet/internal/cdn/cdn_http.go index 1e4e5f8588f15..1bcc8124112f3 100644 --- a/pkg/fleet/internal/cdn/cdn_remote.go +++ b/pkg/fleet/internal/cdn/cdn_http.go @@ -20,12 +20,12 @@ import ( "gopkg.in/DataDog/dd-trace-go.v1/ddtrace/tracer" ) -type cdnRemote struct { +type cdnHTTP struct { client *remoteconfig.HTTPClient currentRootsVersion uint64 } -func newRemote(env *env.Env, configDBPath string) (CDN, error) { +func newCDNHTTP(env *env.Env, configDBPath string) (CDN, error) { client, err := remoteconfig.NewHTTPClient( configDBPath, env.Site, @@ -35,24 +35,24 @@ func newRemote(env *env.Env, configDBPath string) (CDN, error) { if err != nil { return nil, err } - return &cdnRemote{ + return &cdnHTTP{ client: client, currentRootsVersion: 1, }, nil } // Get gets the configuration from the CDN. -func (c *cdnRemote) Get(ctx context.Context, pkg string) (cfg Config, err error) { +func (c *cdnHTTP) Get(ctx context.Context, pkg string) (cfg Config, err error) { span, _ := tracer.StartSpanFromContext(ctx, "cdn.Get") + span.SetTag("cdn_type", "cdn") defer func() { span.Finish(tracer.WithError(err)) }() - orderConfig, layers, err := c.get(ctx) - if err != nil { - return nil, err - } - switch pkg { case "datadog-agent": + orderConfig, layers, err := c.get(ctx) + if err != nil { + return nil, err + } cfg, err = newAgentConfig(orderConfig, layers...) if err != nil { return nil, err @@ -65,12 +65,12 @@ func (c *cdnRemote) Get(ctx context.Context, pkg string) (cfg Config, err error) } // Close cleans up the CDN's resources -func (c *cdnRemote) Close() error { +func (c *cdnHTTP) Close() error { return c.client.Close() } // get calls the Remote Config service to get the ordered layers. -func (c *cdnRemote) get(ctx context.Context) (*orderConfig, [][]byte, error) { +func (c *cdnHTTP) get(ctx context.Context) (*orderConfig, [][]byte, error) { agentConfigUpdate, err := c.client.GetCDNConfigUpdate( ctx, []string{"AGENT_CONFIG"}, diff --git a/pkg/fleet/internal/cdn/cdn_local.go b/pkg/fleet/internal/cdn/cdn_local.go index b6fc898bca66d..b70e0c4499dd8 100644 --- a/pkg/fleet/internal/cdn/cdn_local.go +++ b/pkg/fleet/internal/cdn/cdn_local.go @@ -19,8 +19,8 @@ type cdnLocal struct { dirPath string } -// newLocal creates a new local CDN. -func newLocal(env *env.Env) (CDN, error) { +// newCDNLocal creates a new local CDN. +func newCDNLocal(env *env.Env) (CDN, error) { return &cdnLocal{ dirPath: env.CDNLocalDirPath, }, nil diff --git a/pkg/fleet/internal/cdn/cdn_noop.go b/pkg/fleet/internal/cdn/cdn_noop.go new file mode 100644 index 0000000000000..b18e0e788c011 --- /dev/null +++ b/pkg/fleet/internal/cdn/cdn_noop.go @@ -0,0 +1,43 @@ +// Unless explicitly stated otherwise all files in this repository are licensed +// under the Apache License Version 2.0. +// This product includes software developed at Datadog (https://www.datadoghq.com/). +// Copyright 2016-present Datadog, Inc. + +package cdn + +import ( + "context" + + "github.com/DataDog/datadog-agent/pkg/util/log" +) + +type cdnNoop struct { +} + +type configNoop struct{} + +// newCDNNoop creates a new noop CDN. +func newCDNNoop() (CDN, error) { + return &cdnNoop{}, nil +} + +// Get gets the configuration from the CDN. +func (c *cdnNoop) Get(_ context.Context, _ string) (Config, error) { + log.Debug("Noop CDN get") + return &configNoop{}, nil +} + +func (c *cdnNoop) Close() error { + log.Debug("Noop CDN close") + return nil +} + +func (c *configNoop) Version() string { + log.Debug("Noop CDN version") + return "" +} + +func (c *configNoop) Write(_ string) error { + log.Debug("Noop CDN write") + return nil +} diff --git a/pkg/fleet/internal/cdn/cdn_rc.go b/pkg/fleet/internal/cdn/cdn_rc.go new file mode 100644 index 0000000000000..de3d21c1f11a7 --- /dev/null +++ b/pkg/fleet/internal/cdn/cdn_rc.go @@ -0,0 +1,200 @@ +// Unless explicitly stated otherwise all files in this repository are licensed +// under the Apache License Version 2.0. +// This product includes software developed at Datadog (https://www.datadoghq.com/). +// Copyright 2016-present Datadog, Inc. + +package cdn + +import ( + "context" + "encoding/json" + "fmt" + "os" + "time" + + "github.com/DataDog/datadog-agent/comp/remote-config/rctelemetryreporter/rctelemetryreporterimpl" + remoteconfig "github.com/DataDog/datadog-agent/pkg/config/remote/service" + pkgconfigsetup "github.com/DataDog/datadog-agent/pkg/config/setup" + "github.com/DataDog/datadog-agent/pkg/fleet/env" + pbgo "github.com/DataDog/datadog-agent/pkg/proto/pbgo/core" + pkghostname "github.com/DataDog/datadog-agent/pkg/util/hostname" + "github.com/DataDog/datadog-agent/pkg/version" + "github.com/DataDog/go-tuf/data" + "github.com/google/uuid" + "go.uber.org/multierr" + "gopkg.in/DataDog/dd-trace-go.v1/ddtrace/tracer" +) + +type cdnRC struct { + rcService *remoteconfig.CoreAgentService + currentRootsVersion uint64 + clientUUID string + configDBPath string + firstRequest bool +} + +// newCDNRC creates a new CDN with RC: it fetches the configuration from the remote config service instead of cloudfront +// note: naming is a bit misleading, it's not really a cdn, but we're following the convention +func newCDNRC(env *env.Env, configDBPath string) (CDN, error) { + ctx := context.Background() + ctx, cc := context.WithTimeout(ctx, 10*time.Second) + defer cc() + + ht := newHostTagsGetter() + hostname, err := pkghostname.Get(ctx) + if err != nil { + hostname = "unknown" + } + + // ensures the config db path exists + err = os.MkdirAll(configDBPath, 0755) + if err != nil { + return nil, err + } + + configDBPathTemp, err := os.MkdirTemp(configDBPath, "direct-*") + if err != nil { + return nil, err + } + + options := []remoteconfig.Option{ + remoteconfig.WithAPIKey(env.APIKey), + remoteconfig.WithConfigRootOverride(env.Site, ""), + remoteconfig.WithDirectorRootOverride(env.Site, ""), + remoteconfig.WithDatabaseFileName("remote-config.db"), + remoteconfig.WithDatabasePath(configDBPathTemp), + } + + service, err := remoteconfig.NewService( + pkgconfigsetup.Datadog(), // May not be filled as we don't read the config when we're not in the daemon, in which case we'll use the defaults + "Datadog Installer", + fmt.Sprintf("https://config.%s", env.Site), + hostname, + ht.get, + &rctelemetryreporterimpl.DdRcTelemetryReporter{}, // No telemetry for this client + version.AgentVersion, + options..., + ) + if err != nil { + return nil, err + } + cdn := &cdnRC{ + rcService: service, + currentRootsVersion: 1, + clientUUID: uuid.New().String(), + configDBPath: configDBPathTemp, + firstRequest: true, + } + service.Start() + return cdn, nil +} + +func (c *cdnRC) Get(ctx context.Context, pkg string) (cfg Config, err error) { + span, _ := tracer.StartSpanFromContext(ctx, "cdn.Get") + span.SetTag("cdn_type", "remote_config") + defer func() { span.Finish(tracer.WithError(err)) }() + + switch pkg { + case "datadog-agent": + orderConfig, layers, err := c.get(ctx) + if err != nil { + return nil, err + } + cfg, err = newAgentConfig(orderConfig, layers...) + if err != nil { + return nil, err + } + default: + return nil, ErrProductNotSupported + } + + return cfg, nil +} + +// get calls the Remote Config service to get the ordered layers. +func (c *cdnRC) get(ctx context.Context) (*orderConfig, [][]byte, error) { + if c.firstRequest { + // A first request is made to the remote config service at service startup, + // so if we do another request too close to the first one (in the same second) + // we'll get the same director version (== timestamp) with different contents, + // which will cause the response to be rejected silently and we won't get + // the configurations + time.Sleep(1 * time.Second) + c.firstRequest = false + } + + agentConfigUpdate, err := c.rcService.ClientGetConfigs(ctx, &pbgo.ClientGetConfigsRequest{ + Client: &pbgo.Client{ + Id: c.clientUUID, + Products: []string{"AGENT_CONFIG"}, + IsUpdater: true, + ClientUpdater: &pbgo.ClientUpdater{ + Tags: []string{"installer:true"}, + }, + State: &pbgo.ClientState{ + RootVersion: c.currentRootsVersion, + TargetsVersion: 0, + }, + }, + }) + if err != nil { + return nil, nil, err + } + + if agentConfigUpdate == nil { + return &orderConfig{}, nil, nil + } + + // Update root versions + for _, root := range agentConfigUpdate.Roots { + var signedRoot data.Signed + err = json.Unmarshal(root, &signedRoot) + if err != nil { + continue + } + var r data.Root + err = json.Unmarshal(signedRoot.Signed, &r) + if err != nil { + continue + } + if uint64(r.Version) > c.currentRootsVersion { + c.currentRootsVersion = uint64(r.Version) + } + } + + // Unmarshal RC results + configLayers := make([][]byte, 0) + var configOrder *orderConfig + var layersErr error + for _, file := range agentConfigUpdate.TargetFiles { + matched := datadogConfigIDRegexp.FindStringSubmatch(file.GetPath()) + if len(matched) != 2 { + layersErr = multierr.Append(layersErr, fmt.Errorf("invalid config path: %s", file.GetPath())) + continue + } + configName := matched[1] + + if configName != configOrderID { + configLayers = append(configLayers, file.GetRaw()) + } else { + configOrder = &orderConfig{} + err = json.Unmarshal(file.GetRaw(), configOrder) + if err != nil { + // Return first - we can't continue without the order + return nil, nil, err + } + } + } + if layersErr != nil { + return nil, nil, layersErr + } + return configOrder, configLayers, nil +} + +func (c *cdnRC) Close() error { + err := c.rcService.Stop() + if err != nil { + return err + } + return os.RemoveAll(c.configDBPath) +} diff --git a/pkg/fleet/internal/cdn/tags.go b/pkg/fleet/internal/cdn/tags.go new file mode 100644 index 0000000000000..85f5eb3a931b7 --- /dev/null +++ b/pkg/fleet/internal/cdn/tags.go @@ -0,0 +1,71 @@ +// Unless explicitly stated otherwise all files in this repository are licensed +// under the Apache License Version 2.0. +// This product includes software developed at Datadog (https://www.datadoghq.com/). +// Copyright 2016-present Datadog, Inc. + +package cdn + +import ( + "context" + "os" + "runtime" + "time" + + "github.com/DataDog/datadog-agent/comp/metadata/host/hostimpl/hosttags" + detectenv "github.com/DataDog/datadog-agent/pkg/config/env" + "github.com/DataDog/datadog-agent/pkg/config/model" + pkgconfigsetup "github.com/DataDog/datadog-agent/pkg/config/setup" + "github.com/DataDog/datadog-agent/pkg/util/log" + "gopkg.in/yaml.v2" +) + +type hostTagsGetter struct { + config model.Config +} + +func newHostTagsGetter() hostTagsGetter { + config := pkgconfigsetup.Datadog() + detectenv.DetectFeatures(config) // For host tags to work + err := populateTags(config) + if err != nil { + log.Warnf("Failed to populate tags from datadog.yaml: %v", err) + } + return hostTagsGetter{ + config: config, + } +} + +type tagsConfigFields struct { + Tags []string `yaml:"tags"` + ExtraTags []string `yaml:"extra_tags"` +} + +// populateTags is a best effort to get the tags from `datadog.yaml`. +func populateTags(config model.Config) error { + configPath := "/etc/datadog-agent/datadog.yaml" + if runtime.GOOS == "windows" { + configPath = "C:\\ProgramData\\Datadog\\datadog.yaml" + } + rawConfig, err := os.ReadFile(configPath) + if err != nil { + return err + } + var cfg tagsConfigFields + err = yaml.Unmarshal(rawConfig, &cfg) + if err != nil { + return err + } + config.Set("tags", cfg.Tags, model.SourceFile) + config.Set("extra_tags", cfg.ExtraTags, model.SourceFile) + return nil +} + +func (h *hostTagsGetter) get() []string { + // Host tags are cached on host, but we add a timeout to avoid blocking the request + // if the host tags are not available yet and need to be fetched + ctx, cc := context.WithTimeout(context.Background(), time.Second) + defer cc() + hostTags := hosttags.Get(ctx, true, h.config) + tags := append(hostTags.System, hostTags.GoogleCloudPlatform...) + return tags +} diff --git a/test/new-e2e/tests/installer/host/host.go b/test/new-e2e/tests/installer/host/host.go index 0488a72130557..b736809965ee8 100644 --- a/test/new-e2e/tests/installer/host/host.go +++ b/test/new-e2e/tests/installer/host/host.go @@ -143,7 +143,7 @@ func (h *Host) DeletePath(path string) { func (h *Host) WaitForUnitActive(units ...string) { for _, unit := range units { _, err := h.remote.Execute(fmt.Sprintf("timeout=30; unit=%s; while ! systemctl is-active --quiet $unit && [ $timeout -gt 0 ]; do sleep 1; ((timeout--)); done; [ $timeout -ne 0 ]", unit)) - require.NoError(h.t, err, "unit %s did not become active", unit) + require.NoError(h.t, err, "unit %s did not become active. logs: %s", unit, h.remote.MustExecute("sudo journalctl -xeu "+unit)) } } @@ -151,7 +151,8 @@ func (h *Host) WaitForUnitActive(units ...string) { func (h *Host) WaitForUnitActivating(units ...string) { for _, unit := range units { _, err := h.remote.Execute(fmt.Sprintf("timeout=30; unit=%s; while ! grep -q \"Active: activating\" <(sudo systemctl status $unit) && [ $timeout -gt 0 ]; do sleep 1; ((timeout--)); done; [ $timeout -ne 0 ]", unit)) - require.NoError(h.t, err, "unit %s did not become active", unit) + require.NoError(h.t, err, "unit %s did not become activating. logs: %s", unit, h.remote.MustExecute("sudo journalctl -xeu "+unit)) + } } diff --git a/test/new-e2e/tests/installer/unix/upgrade_scenario_test.go b/test/new-e2e/tests/installer/unix/upgrade_scenario_test.go index 62da6d6ffda4c..eece340b956a8 100644 --- a/test/new-e2e/tests/installer/unix/upgrade_scenario_test.go +++ b/test/new-e2e/tests/installer/unix/upgrade_scenario_test.go @@ -624,7 +624,7 @@ func (s *upgradeScenarioSuite) assertSuccessfulAgentStopExperiment(timestamp hos func (s *upgradeScenarioSuite) startConfigExperiment(localCDNPath string, pkg packageName, hash string) (string, error) { cmd := fmt.Sprintf("sudo -E datadog-installer install-config-experiment %s %s > /tmp/start_config_experiment.log 2>&1", pkg, hash) s.T().Logf("Running start command: %s", cmd) - return s.Env().RemoteHost.Execute(cmd, client.WithEnvVariables(map[string]string{"DD_INSTALLER_DEBUG_CDN_LOCAL_DIR_PATH": localCDNPath})) + return s.Env().RemoteHost.Execute(cmd, client.WithEnvVariables(map[string]string{"DD_INSTALLER_DEBUG_CDN_LOCAL_DIR_PATH": localCDNPath, "DD_REMOTE_POLICIES": "true"})) } func (s *upgradeScenarioSuite) mustStartConfigExperiment(localCDNPath string, pkg packageName, version string) string { @@ -640,7 +640,7 @@ func (s *upgradeScenarioSuite) mustStartConfigExperiment(localCDNPath string, pk func (s *upgradeScenarioSuite) promoteConfigExperiment(localCDNPath string, pkg packageName) (string, error) { cmd := fmt.Sprintf("sudo -E datadog-installer promote-config-experiment %s > /tmp/promote_config_experiment.log 2>&1", pkg) s.T().Logf("Running promote command: %s", cmd) - return s.Env().RemoteHost.Execute(cmd, client.WithEnvVariables(map[string]string{"DD_INSTALLER_DEBUG_CDN_LOCAL_DIR_PATH": localCDNPath})) + return s.Env().RemoteHost.Execute(cmd, client.WithEnvVariables(map[string]string{"DD_INSTALLER_DEBUG_CDN_LOCAL_DIR_PATH": localCDNPath, "DD_REMOTE_POLICIES": "true"})) } func (s *upgradeScenarioSuite) mustPromoteConfigExperiment(localCDNPath string, pkg packageName) string { @@ -656,7 +656,7 @@ func (s *upgradeScenarioSuite) mustPromoteConfigExperiment(localCDNPath string, func (s *upgradeScenarioSuite) stopConfigExperiment(localCDNPath string, pkg packageName) (string, error) { cmd := fmt.Sprintf("sudo -E datadog-installer remove-config-experiment %s > /tmp/stop_config_experiment.log 2>&1", pkg) s.T().Logf("Running stop command: %s", cmd) - return s.Env().RemoteHost.Execute(cmd, client.WithEnvVariables(map[string]string{"DD_INSTALLER_DEBUG_CDN_LOCAL_DIR_PATH": localCDNPath})) + return s.Env().RemoteHost.Execute(cmd, client.WithEnvVariables(map[string]string{"DD_INSTALLER_DEBUG_CDN_LOCAL_DIR_PATH": localCDNPath, "DD_REMOTE_POLICIES": "true"})) } func (s *upgradeScenarioSuite) mustStopConfigExperiment(localCDNPath string, pkg packageName) string { @@ -735,12 +735,20 @@ func (s *upgradeScenarioSuite) assertSuccessfulConfigStopExperiment(timestamp ho func (s *upgradeScenarioSuite) getInstallerStatus() installerStatus { socketPath := "/opt/datadog-packages/run/installer.sock" - requestHeader := " -H 'Content-Type: application/json' -H 'Accept: application/json' " - response := s.Env().RemoteHost.MustExecute(fmt.Sprintf( - "sudo curl -s --unix-socket %s %s http://daemon/status", - socketPath, - requestHeader, - )) + var response string + assert.Eventually(s.T(), func() bool { + var err error + requestHeader := " -H 'Content-Type: application/json' -H 'Accept: application/json' " + response, err = s.Env().RemoteHost.Execute(fmt.Sprintf( + "sudo curl -s --unix-socket %s %s http://daemon/status", + socketPath, + requestHeader, + )) + return err == nil + }, time.Second*30, time.Second*1, "Failed to get installer status: %s\n\n%s", + s.Env().RemoteHost.MustExecute("sudo journalctl -xeu datadog-installer"), + s.Env().RemoteHost.MustExecute("sudo journalctl -xeu datadog-installer-exp"), + ) // {"version":"7.56.0-devel+git.446.acf2836","packages":{ // "datadog-agent":{"Stable":"7.56.0-devel.git.446.acf2836.pipeline.37567760-1","Experiment":"7.54.1-1"},