Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

chore(fleet): Add "direct" RC option to pull configurations #30369

Merged
merged 19 commits into from
Nov 5, 2024
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
22 changes: 11 additions & 11 deletions cmd/installer/subcommands/installer/command.go
Original file line number Diff line number Diff line change
Expand Up @@ -293,7 +293,7 @@ func installCommand() *cobra.Command {
if err != nil {
return err
}
defer i.stop(err)
defer func() { i.stop(err) }()
i.span.SetTag("params.url", args[0])
return i.Install(i.ctx, args[0], installArgs)
},
Expand All @@ -313,7 +313,7 @@ func removeCommand() *cobra.Command {
if err != nil {
return err
}
defer i.stop(err)
defer func() { i.stop(err) }()
i.span.SetTag("params.package", args[0])
return i.Remove(i.ctx, args[0])
},
Expand All @@ -332,7 +332,7 @@ func purgeCommand() *cobra.Command {
if err != nil {
return err
}
defer i.stop(err)
defer func() { i.stop(err) }()
i.Purge(i.ctx)
return nil
},
Expand All @@ -351,7 +351,7 @@ func installExperimentCommand() *cobra.Command {
if err != nil {
return err
}
defer i.stop(err)
defer func() { i.stop(err) }()
i.span.SetTag("params.url", args[0])
return i.InstallExperiment(i.ctx, args[0])
},
Expand All @@ -370,7 +370,7 @@ func removeExperimentCommand() *cobra.Command {
if err != nil {
return err
}
defer i.stop(err)
defer func() { i.stop(err) }()
i.span.SetTag("params.package", args[0])
return i.RemoveExperiment(i.ctx, args[0])
},
Expand All @@ -389,7 +389,7 @@ func promoteExperimentCommand() *cobra.Command {
if err != nil {
return err
}
defer i.stop(err)
defer func() { i.stop(err) }()
i.span.SetTag("params.package", args[0])
return i.PromoteExperiment(i.ctx, args[0])
},
Expand All @@ -408,7 +408,7 @@ func installConfigExperimentCommand() *cobra.Command {
if err != nil {
return err
}
defer func() { i.Stop(err) }()
defer func() { i.stop(err) }()
i.span.SetTag("params.package", args[0])
i.span.SetTag("params.version", args[1])
return i.InstallConfigExperiment(i.ctx, args[0], args[1])
Expand All @@ -428,7 +428,7 @@ func removeConfigExperimentCommand() *cobra.Command {
if err != nil {
return err
}
defer func() { i.Stop(err) }()
defer func() { i.stop(err) }()
i.span.SetTag("params.package", args[0])
return i.RemoveConfigExperiment(i.ctx, args[0])
},
Expand All @@ -447,7 +447,7 @@ func promoteConfigExperimentCommand() *cobra.Command {
if err != nil {
return err
}
defer func() { i.Stop(err) }()
defer func() { i.stop(err) }()
i.span.SetTag("params.package", args[0])
return i.PromoteConfigExperiment(i.ctx, args[0])
},
Expand All @@ -466,7 +466,7 @@ func garbageCollectCommand() *cobra.Command {
if err != nil {
return err
}
defer i.stop(err)
defer func() { i.stop(err) }()
return i.GarbageCollect(i.ctx)
},
}
Expand All @@ -489,7 +489,7 @@ func isInstalledCommand() *cobra.Command {
if err != nil {
return err
}
defer i.stop(err)
defer func() { i.stop(err) }()
installed, err := i.IsInstalled(i.ctx, args[0])
if err != nil {
return err
Expand Down
18 changes: 15 additions & 3 deletions pkg/config/remote/service/service.go
Original file line number Diff line number Diff line change
Expand Up @@ -17,14 +17,15 @@ import (
"errors"
"expvar"
"fmt"
"github.com/DataDog/datadog-agent/pkg/remoteconfig/state"
"gopkg.in/DataDog/dd-trace-go.v1/ddtrace/tracer"
"net/url"
"path"
"strconv"
"sync"
"time"

"github.com/DataDog/datadog-agent/pkg/remoteconfig/state"
"gopkg.in/DataDog/dd-trace-go.v1/ddtrace/tracer"

"github.com/DataDog/go-tuf/data"
tufutil "github.com/DataDog/go-tuf/util"
"github.com/benbjohnson/clock"
Expand Down Expand Up @@ -247,6 +248,7 @@ type options struct {
apiKey string
traceAgentEnv string
databaseFileName string
databaseFilePath string
configRootOverride string
directorRootOverride string
clientCacheBypassLimit int
Expand All @@ -261,6 +263,7 @@ var defaultOptions = options{
apiKey: "",
traceAgentEnv: "",
databaseFileName: "remote-config.db",
databaseFilePath: "",
configRootOverride: "",
directorRootOverride: "",
clientCacheBypassLimit: defaultCacheBypassLimit,
Expand All @@ -283,6 +286,11 @@ func WithDatabaseFileName(fileName string) func(s *options) {
return func(s *options) { s.databaseFileName = fileName }
}

// WithDatabasePath sets the service database path
func WithDatabasePath(path string) func(s *options) {
return func(s *options) { s.databaseFilePath = path }
}

// WithConfigRootOverride sets the service config root override
func WithConfigRootOverride(site string, override string) func(s *options) {
return func(opts *options) {
Expand Down Expand Up @@ -410,7 +418,11 @@ func NewService(cfg model.Reader, rcType, baseRawURL, hostname string, tagsGette
return nil, err
}

dbPath := path.Join(cfg.GetString("run_path"), options.databaseFileName)
databaseFilePath := cfg.GetString("run_path")
if options.databaseFilePath != "" {
databaseFilePath = options.databaseFilePath
}
dbPath := path.Join(databaseFilePath, options.databaseFileName)
db, err := openCacheDB(dbPath, agentVersion, authKeys.apiKey)
if err != nil {
return nil, err
Expand Down
7 changes: 2 additions & 5 deletions pkg/fleet/daemon/daemon.go
Original file line number Diff line number Diff line change
Expand Up @@ -41,7 +41,7 @@ const (
// gcInterval is the interval at which the GC will run
gcInterval = 1 * time.Hour
// refreshStateInterval is the interval at which the state will be refreshed
refreshStateInterval = 1 * time.Minute
refreshStateInterval = 30 * time.Second
BaptisteFoy marked this conversation as resolved.
Show resolved Hide resolved
)

// Daemon is the fleet daemon in charge of remote install, updates and configuration.
Expand Down Expand Up @@ -562,7 +562,7 @@ func (d *daemonImpl) resolveRemoteConfigVersion(ctx context.Context, pkg string)
}
config, err := d.cdn.Get(ctx, pkg)
if err != nil {
return "", fmt.Errorf("could not get cdn config: %w", err)
return "", err
}
return config.Version(), nil
}
Expand Down Expand Up @@ -602,9 +602,6 @@ func (d *daemonImpl) refreshState(ctx context.Context) {
}

configVersion, err := d.resolveRemoteConfigVersion(ctx, pkg)
if err != nil {
log.Errorf("could not get agent remote config version: %v", err)
}
if err == nil {
p.RemoteConfigVersion = configVersion
} else if err != cdn.ErrProductNotSupported {
BaptisteFoy marked this conversation as resolved.
Show resolved Hide resolved
Expand Down
3 changes: 3 additions & 0 deletions pkg/fleet/env/env.go
Original file line number Diff line number Diff line change
Expand Up @@ -32,6 +32,7 @@ const (
envAgentMinorVersion = "DD_AGENT_MINOR_VERSION"
envApmLanguages = "DD_APM_INSTRUMENTATION_LANGUAGES"
envCDNLocalDirPath = "DD_INSTALLER_DEBUG_CDN_LOCAL_DIR_PATH"
envCDNEnabled = "DD_INSTALLER_CDN_ENABLED"
)

var defaultEnv = Env{
Expand Down Expand Up @@ -88,6 +89,7 @@ type Env struct {

InstallScript InstallScriptEnv

CDNEnabled bool
CDNLocalDirPath string
}

Expand Down Expand Up @@ -118,6 +120,7 @@ func FromEnv() *Env {

InstallScript: installScriptEnvFromEnv(),

CDNEnabled: strings.ToLower(os.Getenv(envCDNEnabled)) == "true",
CDNLocalDirPath: getEnvOrDefault(envCDNLocalDirPath, ""),
}
}
Expand Down
33 changes: 30 additions & 3 deletions pkg/fleet/internal/cdn/cdn.go
Original file line number Diff line number Diff line change
Expand Up @@ -10,6 +10,7 @@ import (
"context"
"errors"
"regexp"
"runtime"

"github.com/DataDog/datadog-agent/pkg/fleet/env"
)
Expand Down Expand Up @@ -38,10 +39,36 @@ type CDN interface {
Close() error
}

// New creates a new CDN.
// New creates a new CDN and chooses the implementation depending
// on the environment
func New(env *env.Env, configDBPath string) (CDN, error) {
if runtime.GOOS == "windows" {
BaptisteFoy marked this conversation as resolved.
Show resolved Hide resolved
// There's an assumption on windows that some directories are already there
// but they are in fact created by the regular CDN implementation. Until
// there is a fix on windows we keep the previous CDN behaviour for them
return newCDNHTTP(env, configDBPath)
}

if !env.RemotePolicies {
// Remote policies are not enabled -- we don't need the CDN
// and we don't want to create the directories that the CDN
// implementation would create. We return a no-op CDN to avoid
// nil pointer dereference.
return newCDNNoop()
}

if env.CDNLocalDirPath != "" {
return newLocal(env)
// Mock the CDN for local development or testing
return newCDNLocal(env)
}
return newRemote(env, configDBPath)

if !env.CDNEnabled {
// Remote policies are enabled but we don't want to use the CDN
// as it's still in development. We use standard remote config calls
// instead (dubbed "direct" CDN).
return newCDNRC(env, configDBPath)
}

// Regular CDN with the cloudfront distribution
return newCDNHTTP(env, configDBPath)
}
Original file line number Diff line number Diff line change
Expand Up @@ -20,12 +20,12 @@ import (
"gopkg.in/DataDog/dd-trace-go.v1/ddtrace/tracer"
)

type cdnRemote struct {
type cdnHTTP struct {
client *remoteconfig.HTTPClient
currentRootsVersion uint64
}

func newRemote(env *env.Env, configDBPath string) (CDN, error) {
func newCDNHTTP(env *env.Env, configDBPath string) (CDN, error) {
client, err := remoteconfig.NewHTTPClient(
configDBPath,
env.Site,
Expand All @@ -35,24 +35,24 @@ func newRemote(env *env.Env, configDBPath string) (CDN, error) {
if err != nil {
return nil, err
}
return &cdnRemote{
return &cdnHTTP{
client: client,
currentRootsVersion: 1,
}, nil
}

// Get gets the configuration from the CDN.
func (c *cdnRemote) Get(ctx context.Context, pkg string) (cfg Config, err error) {
func (c *cdnHTTP) Get(ctx context.Context, pkg string) (cfg Config, err error) {
span, _ := tracer.StartSpanFromContext(ctx, "cdn.Get")
span.SetTag("cdn_type", "cdn")
defer func() { span.Finish(tracer.WithError(err)) }()

orderConfig, layers, err := c.get(ctx)
if err != nil {
return nil, err
}

switch pkg {
case "datadog-agent":
orderConfig, layers, err := c.get(ctx)
if err != nil {
return nil, err
}
cfg, err = newAgentConfig(orderConfig, layers...)
if err != nil {
return nil, err
Expand All @@ -65,12 +65,12 @@ func (c *cdnRemote) Get(ctx context.Context, pkg string) (cfg Config, err error)
}

// Close cleans up the CDN's resources
func (c *cdnRemote) Close() error {
func (c *cdnHTTP) Close() error {
return c.client.Close()
}

// get calls the Remote Config service to get the ordered layers.
func (c *cdnRemote) get(ctx context.Context) (*orderConfig, [][]byte, error) {
func (c *cdnHTTP) get(ctx context.Context) (*orderConfig, [][]byte, error) {
agentConfigUpdate, err := c.client.GetCDNConfigUpdate(
ctx,
[]string{"AGENT_CONFIG"},
Expand Down
4 changes: 2 additions & 2 deletions pkg/fleet/internal/cdn/cdn_local.go
Original file line number Diff line number Diff line change
Expand Up @@ -19,8 +19,8 @@ type cdnLocal struct {
dirPath string
}

// newLocal creates a new local CDN.
func newLocal(env *env.Env) (CDN, error) {
// newCDNLocal creates a new local CDN.
func newCDNLocal(env *env.Env) (CDN, error) {
return &cdnLocal{
dirPath: env.CDNLocalDirPath,
}, nil
Expand Down
43 changes: 43 additions & 0 deletions pkg/fleet/internal/cdn/cdn_noop.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,43 @@
// Unless explicitly stated otherwise all files in this repository are licensed
// under the Apache License Version 2.0.
// This product includes software developed at Datadog (https://www.datadoghq.com/).
// Copyright 2016-present Datadog, Inc.

package cdn

import (
"context"

"github.com/DataDog/datadog-agent/pkg/util/log"
)

type cdnNoop struct {
}

type configNoop struct{}

// newCDNNoop creates a new noop CDN.
func newCDNNoop() (CDN, error) {
return &cdnNoop{}, nil
}

// Get gets the configuration from the CDN.
func (c *cdnNoop) Get(_ context.Context, _ string) (Config, error) {
log.Debug("Noop CDN get")
return &configNoop{}, nil
}

func (c *cdnNoop) Close() error {
log.Debug("Noop CDN close")
return nil
BaptisteFoy marked this conversation as resolved.
Show resolved Hide resolved
}

func (c *configNoop) Version() string {
log.Debug("Noop CDN version")
return ""
}

func (c *configNoop) Write(_ string) error {
log.Debug("Noop CDN write")
return nil
}
Loading
Loading