Merge branch 'main' into session_view_processor_ebpf

CHANGELOG-developer.next.asciidoc

@@ -87,6 +87,7 @@ The list below covers the major changes between 7.0.0-rc2 and main only.
 - Fix ingest pipeline for panw module to parse url scheme correctly {pull}35757[35757]
 - Renamed an httpjson input metric to follow naming conventions. `httpjson_interval_pages_total` was renamed to `httpjson_interval_pages` because the `_total` suffix is reserved for counters. {issue}35933[35933] {pull}36169[36169]
 - Fixed some race conditions in tests {pull}36185[36185]
+- Fix Stringer implementation of fingerprint processor {issue}35174[35174]
 - Re-enable HTTPJSON fixed flakey test. {issue}34929[34929] {pull}36525[36525]
 - Make winlogbeat/sys/wineventlog follow the unsafe.Pointer rules. {pull}36650[36650]
 - Cleaned up documentation errors & fixed a minor bug in Filebeat Azure blob storage input. {pull}36714[36714]

CHANGELOG.next.asciidoc

@@ -15,6 +15,9 @@ https://github.com/elastic/beats/compare/v8.8.1\...main[Check the HEAD diff]
 platform, and when viewed from a metadata API standpoint, it is impossible to differentiate it from OpenStack. If you
 know that your deployments run on Huawei Cloud exclusively, and you wish to have `cloud.provider` value as `huawei`,
 you can achieve this by overwriting the value using an `add_fields` processor. {pull}35184[35184]
+ - In managed mode, Beats running under Elastic Agent will report the package
+version of Elastic Agent as their own version. This includes all additional
+fields added to events containing the Beats version. {pull}37553[37553]
 
 *Auditbeat*
 
@@ -62,6 +65,7 @@ you can achieve this by overwriting the value using an `add_fields` processor. {
 - Upgrade elastic-agent-libs to v0.7.5. Removes obsolete "Treating the CommonName field on X.509 certificates as a host name..." deprecation warning for 8.0. {pull}37755[37755]
 - aws: Add credential caching for `AssumeRole` session tokens. {issue}37787[37787]
 - Lower logging level to debug when attempting to configure beats with unknown fields from autodiscovered events/environments {pull}[37816][37816]
+- Set timeout of 1 minute for FQDN requests {pull}37756[37756]
 
 *Auditbeat*
 
@@ -184,6 +188,7 @@ Setting environmental variable ELASTIC_NETINFO:false in Elastic Agent pod will d
 - Add request trace logging for chained API requests. {issue}37551[36551] {pull}37682[37682]
 - Relax TCP/UDP metric polling expectations to improve metric collection. {pull}37714[37714]
 - Add support for PEM-based Okta auth in HTTPJSON. {pull}37772[37772]
+- Prevent complete loss of long request trace data. {issue}37826[37826] {pull}37836[37836]
 
 *Auditbeat*
 

NOTICE.txt

@@ -12494,11 +12494,11 @@ Contents of probable licence file $GOMODCACHE/github.com/elastic/elastic-agent-a
 
 --------------------------------------------------------------------------------
 Dependency : github.com/elastic/elastic-agent-client/v7
-Version: v7.6.0
+Version: v7.8.0
 Licence type (autodetected): Elastic
 --------------------------------------------------------------------------------
 
-Contents of probable licence file $GOMODCACHE/github.com/elastic/elastic-agent-client/v7@v7.6.0/LICENSE.txt:
+Contents of probable licence file $GOMODCACHE/github.com/elastic/elastic-agent-client/v7@v7.8.0/LICENSE.txt:
 
 ELASTIC LICENSE AGREEMENT
 
@@ -14981,11 +14981,11 @@ Contents of probable licence file $GOMODCACHE/github.com/elastic/go-structform@v
 
 --------------------------------------------------------------------------------
 Dependency : github.com/elastic/go-sysinfo
-Version: v1.11.2
+Version: v1.12.0
 Licence type (autodetected): Apache-2.0
 --------------------------------------------------------------------------------
 
-Contents of probable licence file $GOMODCACHE/github.com/elastic/go-sysinfo@v1.11.2/LICENSE.txt:
+Contents of probable licence file $GOMODCACHE/github.com/elastic/go-sysinfo@v1.12.0/LICENSE.txt:
 
 
                                  Apache License
@@ -25616,11 +25616,11 @@ Contents of probable licence file $GOMODCACHE/google.golang.org/[email protected]/LIC
 
 --------------------------------------------------------------------------------
 Dependency : google.golang.org/protobuf
-Version: v1.31.0
+Version: v1.32.0
 Licence type (autodetected): BSD-3-Clause
 --------------------------------------------------------------------------------
 
-Contents of probable licence file $GOMODCACHE/google.golang.org/protobuf@v1.31.0/LICENSE:
+Contents of probable licence file $GOMODCACHE/google.golang.org/protobuf@v1.32.0/LICENSE:
 
 Copyright (c) 2018 The Go Authors. All rights reserved.
 

docs/devguide/testing.asciidoc

@@ -50,24 +50,28 @@ In Metricbeat, run the command from within a module like this: `go test --tags i
 
 A note about tags: the `--data` flag is a custom flag added by Metricbeat and Packetbeat frameworks. It will not be present in case tags do not match, as the relevant code will not be run and silently skipped (without the tag the test file is ignored by Go compiler so the framework doesn't load). This may happen if there are different tags in the build tags of the metricset under test (i.e. the GCP billing metricset requires the `billing` tag too).
 
-==== Running Python Tests
+==== Running System (integration) Tests (Python and Go)
 
-Python system tests are defined in the `tests/system` directory. They require a testing binary to be available and the python environment to be set up.
+The system tests are defined in the `tests/system` (for legacy Python test) and on `tests/integration` (for Go tests) directory. They require a testing binary to be available and the python environment to be set up.
 
-To create the testing binary run `mage buildSystemTestBinary`. This will create the test binary in the beat directory. To setup the testing environment run `mage pythonVirtualEnv` which will create a virtual environment with all test dependencies and print its location. To activate it, the instructions depend on your operating system. See the https://packaging.python.org/en/latest/guides/installing-using-pip-and-virtual-environments/#activating-a-virtual-environment[virtualenv documentation].
+To create the testing binary run `mage buildSystemTestBinary`. This will create the test binary in the beat directory. To set up the Python testing environment run `mage pythonVirtualEnv` which will create a virtual environment with all test dependencies and print its location. To activate it, the instructions depend on your operating system. See the https://packaging.python.org/en/latest/guides/installing-using-pip-and-virtual-environments/#activating-a-virtual-environment[virtualenv documentation].
 
 To run the system and integration tests use the `mage pythonIntegTest` target, which will start the required services using https://docs.docker.com/compose/[docker-compose] and run all integration tests. Similar to Go integration tests, the individual steps can be done manually to allow selecting which tests should be run:
 
 [source,bash]
 ----
 # Create and activate the system test virtual environment (assumes a Unix system).
 source $(mage pythonVirtualEnv)/bin/activate
+
 # Pull and build the containers. Only needs to be done once unless you change the containers.
 mage docker:composeBuild
+
 # Bring up all containers, wait until they are healthy, and put them in the background.
 mage docker:composeUp
+
 # Run all system and integration tests.
 INTEGRATION_TESTS=1 pytest ./tests/system
+
 # Stop all started containers.
 mage docker:composeDown
 ----

go.mod

@@ -69,7 +69,7 @@ require (
 	github.com/dustin/go-humanize v1.0.1
 	github.com/eapache/go-resiliency v1.2.0
 	github.com/eclipse/paho.mqtt.golang v1.3.5
-	github.com/elastic/elastic-agent-client/v7 v7.6.0
+	github.com/elastic/elastic-agent-client/v7 v7.8.0
 	github.com/elastic/go-concert v0.2.0
 	github.com/elastic/go-libaudit/v2 v2.5.0
 	github.com/elastic/go-licenser v0.4.1
@@ -78,7 +78,7 @@ require (
 	github.com/elastic/go-perf v0.0.0-20191212140718-9c656876f595
 	github.com/elastic/go-seccomp-bpf v1.4.0
 	github.com/elastic/go-structform v0.0.10
-	github.com/elastic/go-sysinfo v1.11.2
+	github.com/elastic/go-sysinfo v1.12.0
 	github.com/elastic/go-ucfg v0.8.6
 	github.com/elastic/gosigar v0.14.2
 	github.com/fatih/color v1.15.0
@@ -164,7 +164,7 @@ require (
 	google.golang.org/api v0.128.0
 	google.golang.org/genproto v0.0.0-20230920204549-e6e6cdab5c13 // indirect
 	google.golang.org/grpc v1.58.3
-	google.golang.org/protobuf v1.31.0
+	google.golang.org/protobuf v1.32.0
 	gopkg.in/inf.v0 v0.9.1
 	gopkg.in/jcmturner/aescts.v1 v1.0.1 // indirect
 	gopkg.in/jcmturner/dnsutils.v1 v1.0.1 // indirect

go.sum

@@ -663,8 +663,8 @@ github.com/elastic/ebpfevents v0.3.2 h1:UJ8kW5jw2TpUR5MEMaZ1O62sK9JQ+5xTlj+YpQC6
 github.com/elastic/ebpfevents v0.3.2/go.mod h1:o21z5xup/9dK8u0Hg9bZRflSqqj1Zu5h2dg2hSTcUPQ=
 github.com/elastic/elastic-agent-autodiscover v0.6.7 h1:+KVjltN0rPsBrU8b156gV4lOTBgG/vt0efFCFARrf3g=
 github.com/elastic/elastic-agent-autodiscover v0.6.7/go.mod h1:hFeFqneS2r4jD0/QzGkrNk0YVdN0JGh7lCWdsH7zcI4=
-github.com/elastic/elastic-agent-client/v7 v7.6.0 h1:FEn6FjzynW4TIQo5G096Tr7xYK/P5LY9cSS6wRbXZTc=
-github.com/elastic/elastic-agent-client/v7 v7.6.0/go.mod h1:GlUKrbVd/O1CRAZonpBeN3J0RlVqP6VGcrBjFWca+aM=
+github.com/elastic/elastic-agent-client/v7 v7.8.0 h1:GHFzDJIWpdgI0qDk5EcqbQJGvwTsl2E2vQK3/xe+MYQ=
+github.com/elastic/elastic-agent-client/v7 v7.8.0/go.mod h1:ihtjqJzYiIltlRhNruaSSc0ogxIhqPD5hOMKq16cI1s=
 github.com/elastic/elastic-agent-libs v0.7.5 h1:4UMqB3BREvhwecYTs/L23oQp1hs/XUkcunPlmTZn5yg=
 github.com/elastic/elastic-agent-libs v0.7.5/go.mod h1:pGMj5myawdqu+xE+WKvM5FQzKQ/MonikkWOzoFTJxaU=
 github.com/elastic/elastic-agent-shipper-client v0.5.1-0.20230228231646-f04347b666f3 h1:sb+25XJn/JcC9/VL8HX4r4QXSUq4uTNzGS2kxOE7u1U=
@@ -696,8 +696,8 @@ github.com/elastic/go-seccomp-bpf v1.4.0 h1:6y3lYrEHrLH9QzUgOiK8WDqmPaMnnB785Wxi
 github.com/elastic/go-seccomp-bpf v1.4.0/go.mod h1:wIMxjTbKpWGQk4CV9WltlG6haB4brjSH/dvAohBPM1I=
 github.com/elastic/go-structform v0.0.10 h1:oy08o/Ih2hHTkNcRY/1HhaYvIp5z6t8si8gnCJPDo1w=
 github.com/elastic/go-structform v0.0.10/go.mod h1:CZWf9aIRYY5SuKSmOhtXScE5uQiLZNqAFnwKR4OrIM4=
-github.com/elastic/go-sysinfo v1.11.2 h1:mcm4OSYVMyws6+n2HIVMGkln5HOpo5Ie1ZmbbNn0jg4=
-github.com/elastic/go-sysinfo v1.11.2/go.mod h1:GKqR8bbMK/1ITnez9NIsIfXQr25aLhRJa7AfT8HpBFQ=
+github.com/elastic/go-sysinfo v1.12.0 h1:ZKyB4N5XLnGFysNGNnJl8xvd+GBGCe2MemBykR+3yQI=
+github.com/elastic/go-sysinfo v1.12.0/go.mod h1:GKqR8bbMK/1ITnez9NIsIfXQr25aLhRJa7AfT8HpBFQ=
 github.com/elastic/go-ucfg v0.8.6 h1:stUeyh2goTgGX+/wb9gzKvTv0YB0231LTpKUgCKj4U0=
 github.com/elastic/go-ucfg v0.8.6/go.mod h1:4E8mPOLSUV9hQ7sgLEJ4bvt0KhMuDJa8joDT2QGAEKA=
 github.com/elastic/go-windows v1.0.1 h1:AlYZOldA+UJ0/2nBuqWdo90GFCgG9xuyw9SYzGUtJm0=
@@ -2658,8 +2658,8 @@ google.golang.org/protobuf v1.26.0-rc.1/go.mod h1:jlhhOSvTdKEhbULTjvd4ARK9grFBp0
 google.golang.org/protobuf v1.26.0/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc=
 google.golang.org/protobuf v1.27.1/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc=
 google.golang.org/protobuf v1.28.1/go.mod h1:HV8QOd/L58Z+nl8r43ehVNZIU/HEI6OcFqwMG9pJV4I=
-google.golang.org/protobuf v1.31.0 h1:g0LDEJHgrBl9N9r17Ru3sqWhkIx2NB67okBHPwC7hs8=
-google.golang.org/protobuf v1.31.0/go.mod h1:HV8QOd/L58Z+nl8r43ehVNZIU/HEI6OcFqwMG9pJV4I=
+google.golang.org/protobuf v1.32.0 h1:pPC6BG5ex8PDFnkbrGU3EixyhKcQ2aDuBS36lqK/C7I=
+google.golang.org/protobuf v1.32.0/go.mod h1:c6P6GXX6sHbq/GpV6MGZEdwhWPcYBgnhAHhKbcUYpos=
 gopkg.in/airbrake/gobrake.v2 v2.0.9/go.mod h1:/h5ZAUhDkGaJfjzjKLSjv6zCL6O0LLBxU4K+aSYdM/U=
 gopkg.in/alecthomas/kingpin.v2 v2.2.6/go.mod h1:FMv+mEhP44yOT+4EoQTLFTRgOQ1FBLkstjWtayDeSgw=
 gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=

libbeat/cmd/instance/beat.go

@@ -197,7 +197,7 @@ func initRand() {
 	} else {
 		seed = n.Int64()
 	}
-	rand.Seed(seed)
+	rand.Seed(seed) //nolint:staticcheck // need seed from cryptographically strong PRNG.
 }
 
 // Run initializes and runs a Beater implementation. name is the name of the
@@ -824,7 +824,10 @@ func (b *Beat) configure(settings Settings) error {
 		return fmt.Errorf("failed to get host information: %w", err)
 	}
 
-	fqdn, err := h.FQDN()
+	fqdnLookupCtx, cancel := context.WithTimeout(context.Background(), 1*time.Minute)
+	defer cancel()
+
+	fqdn, err := h.FQDNWithContext(fqdnLookupCtx)
 	if err != nil {
 		// FQDN lookup is "best effort".  We log the error, fallback to
 		// the OS-reported hostname, and move on.
@@ -835,10 +838,25 @@ func (b *Beat) configure(settings Settings) error {
 	}
 
 	// initialize config manager
-	b.Manager, err = management.NewManager(b.Config.Management, reload.RegisterV2)
+	m, err := management.NewManager(b.Config.Management, reload.RegisterV2)
 	if err != nil {
 		return err
 	}
+	b.Manager = m
+
+	if b.Manager.AgentInfo().Version != "" {
+		// During the manager initialization the client to connect to the agent is
+		// also initialized. That makes the beat to read information sent by the
+		// agent, which includes the AgentInfo with the agent's package version.
+		// Components running under agent should report the agent's package version
+		// as their own version.
+		// In order to do so b.Info.Version needs to be set to the version the agent
+		// sent. As this Beat instance is initialized much before the package
+		// version is received, it's overridden here. So far it's early enough for
+		// the whole beat to report the right version.
+		b.Info.Version = b.Manager.AgentInfo().Version
+		version.SetPackageVersion(b.Info.Version)
+	}
 
 	if err := b.Manager.CheckRawConfig(b.RawConfig); err != nil {
 		return err
@@ -1518,13 +1536,13 @@ func (bc *beatConfig) Validate() error {
 		if bc.Pipeline.Queue.IsSet() && outputPC.Queue.IsSet() {
 			return fmt.Errorf("top level queue and output level queue settings defined, only one is allowed")
 		}
-		//elastic-agent doesn't support disk queue yet
+		// elastic-agent doesn't support disk queue yet
 		if bc.Management.Enabled() && outputPC.Queue.Config().Enabled() && outputPC.Queue.Name() == diskqueue.QueueType {
 			return fmt.Errorf("disk queue is not supported when management is enabled")
 		}
 	}
 
-	//elastic-agent doesn't support disk queue yet
+	// elastic-agent doesn't support disk queue yet
 	if bc.Management.Enabled() && bc.Pipeline.Queue.Config().Enabled() && bc.Pipeline.Queue.Name() == diskqueue.QueueType {
 		return fmt.Errorf("disk queue is not supported when management is enabled")
 	}

libbeat/management/management.go

@@ -82,9 +82,12 @@ type Manager interface {
 	//
 	// Calls to 'CheckRawConfig()' or 'SetPayload()' will be ignored after calling stop.
 	//
-	// Note: Stop will not call 'UnregisterAction()' automaticallty.
+	// Note: Stop will not call 'UnregisterAction()' automatically.
 	Stop()
 
+	// AgentInfo returns the information of the agent to which the manager is connected.
+	AgentInfo() client.AgentInfo
+
 	// SetStopCallback accepts a function that need to be called when the manager want to shutdown the
 	// beats. This is needed when you want your beats to be gracefully shutdown remotely by the Elastic Agent
 	// when a policy doesn't need to run this beat.
@@ -190,6 +193,7 @@ func (n *fallbackManager) Stop() {
 // but that does not mean the Beat is being managed externally,
 // hence it will always return false.
 func (n *fallbackManager) Enabled() bool                         { return false }
+func (n *fallbackManager) AgentInfo() client.AgentInfo           { return client.AgentInfo{} }
 func (n *fallbackManager) Start() error                          { return nil }
 func (n *fallbackManager) CheckRawConfig(cfg *config.C) error    { return nil }
 func (n *fallbackManager) RegisterAction(action client.Action)   {}

libbeat/processors/add_host_metadata/add_host_metadata.go

@@ -18,13 +18,15 @@
 package add_host_metadata
 
 import (
+	"context"
 	"fmt"
 	"sync"
 	"time"
 
 	"github.com/gofrs/uuid"
 
 	"github.com/elastic/elastic-agent-libs/monitoring"
+	"github.com/elastic/go-sysinfo"
 
 	"github.com/elastic/beats/v7/libbeat/beat"
 	"github.com/elastic/beats/v7/libbeat/features"
@@ -35,7 +37,6 @@ import (
 	"github.com/elastic/elastic-agent-libs/logp"
 	"github.com/elastic/elastic-agent-libs/mapstr"
 	"github.com/elastic/elastic-agent-system-metrics/metric/system/host"
-	"github.com/elastic/go-sysinfo"
 )
 
 const processorName = "add_host_metadata"
@@ -96,7 +97,7 @@ func New(cfg *config.C) (beat.Processor, error) {
 	}
 
 	// create a unique ID for this instance of the processor
-	cbIDStr := ""
+	var cbIDStr string
 	cbID, err := uuid.NewV4()
 	// if we fail, fall back to the processor name, hope for the best.
 	if err != nil {
@@ -178,7 +179,10 @@ func (p *addHostMetadata) loadData(checkCache bool, useFQDN bool) error {
 
 	hostname := h.Info().Hostname
 	if useFQDN {
-		fqdn, err := h.FQDN()
+		ctx, cancel := context.WithTimeout(context.Background(), 1*time.Minute)
+		defer cancel()
+
+		fqdn, err := h.FQDNWithContext(ctx)
 		if err != nil {
 			// FQDN lookup is "best effort". If it fails, we monitor the failure, fallback to
 			// the OS-reported hostname, and move on.

libbeat/processors/fingerprint/config.go

@@ -17,13 +17,15 @@
 
 package fingerprint
 
+import "encoding/json"
+
 // Config for fingerprint processor.
 type Config struct {
-	Method        hashMethod     `config:"method"`                     // Hash function to use for fingerprinting
-	Fields        []string       `config:"fields" validate:"required"` // Source fields to compute fingerprint from
-	TargetField   string         `config:"target_field"`               // Target field for the fingerprint
-	Encoding      encodingMethod `config:"encoding"`                   // Encoding to use for target field value
-	IgnoreMissing bool           `config:"ignore_missing"`             // Ignore missing fields?
+	Method        namedHashMethod     `config:"method"`                     // Hash function to use for fingerprinting
+	Fields        []string            `config:"fields" validate:"required"` // Source fields to compute fingerprint from
+	TargetField   string              `config:"target_field"`               // Target field for the fingerprint
+	Encoding      namedEncodingMethod `config:"encoding"`                   // Encoding to use for target field value
+	IgnoreMissing bool                `config:"ignore_missing"`             // Ignore missing fields?
 }
 
 func defaultConfig() Config {
@@ -34,3 +36,16 @@ func defaultConfig() Config {
 		IgnoreMissing: false,
 	}
 }
+
+func (c *Config) MarshalJSON() ([]byte, error) {
+	type Alias Config
+	return json.Marshal(&struct {
+		Method   string
+		Encoding string
+		*Alias
+	}{
+		Method:   c.Method.Name,
+		Encoding: c.Encoding.Name,
+		Alias:    (*Alias)(c),
+	})
+}

libbeat/processors/fingerprint/encode.go

@@ -24,16 +24,26 @@ import (
 	"strings"
 )
 
+type namedEncodingMethod struct {
+	Name   string
+	Encode encodingMethod
+}
 type encodingMethod func([]byte) string
 
-var encodings = map[string]encodingMethod{
-	"hex":    hex.EncodeToString,
-	"base32": base32.StdEncoding.EncodeToString,
-	"base64": base64.StdEncoding.EncodeToString,
+var encodings = map[string]namedEncodingMethod{}
+
+func init() {
+	for _, e := range []namedEncodingMethod{
+		{Name: "hex", Encode: hex.EncodeToString},
+		{Name: "base32", Encode: base32.StdEncoding.EncodeToString},
+		{Name: "base64", Encode: base64.StdEncoding.EncodeToString},
+	} {
+		encodings[e.Name] = e
+	}
 }
 
 // Unpack creates the encodingMethod from the given string
-func (e *encodingMethod) Unpack(str string) error {
+func (e *namedEncodingMethod) Unpack(str string) error {
 	str = strings.ToLower(str)
 
 	m, found := encodings[str]

libbeat/processors/fingerprint/fingerprint.go

@@ -60,7 +60,7 @@ func New(cfg *config.C) (beat.Processor, error) {
 
 	p := &fingerprint{
 		config: config,
-		hash:   config.Method,
+		hash:   config.Method.Hash,
 		fields: fields,
 	}
 
@@ -75,7 +75,7 @@ func (p *fingerprint) Run(event *beat.Event) (*beat.Event, error) {
 		return nil, makeErrComputeFingerprint(err)
 	}
 
-	encodedHash := p.config.Encoding(hashFn.Sum(nil))
+	encodedHash := p.config.Encoding.Encode(hashFn.Sum(nil))
 
 	if _, err := event.PutValue(p.config.TargetField, encodedHash); err != nil {
 		return nil, makeErrComputeFingerprint(err)
@@ -85,8 +85,7 @@ func (p *fingerprint) Run(event *beat.Event) (*beat.Event, error) {
 }
 
 func (p *fingerprint) String() string {
-	//nolint:staticcheck // https://github.com/elastic/beats/issues/35174
-	json, _ := json.Marshal(p.config)
+	json, _ := json.Marshal(&p.config)
 	return procName + "=" + string(json)
 }