From f7f4a1f36d32aee2ce2e364a0aa1564474df6d6d Mon Sep 17 00:00:00 2001 From: Tiago Queiroz Date: Thu, 30 Jan 2025 19:10:44 -0500 Subject: [PATCH 1/5] Update Journald fields to better match ECS (#42403) The fields produced by the Journald input are updated to better match ECS. Renamed fields: Dropped fields: `syslog.priority` and `syslog.facility` while keeping their duplicated equivalent: `log.syslog.priority`,`log.syslog.facility.code`. Renamed fields: `syslog.identifier` -> `log.syslog.appname`, `syslog.pid` -> `log.syslog.procid`. `container.id_truncated` is dropped because the full container ID is already present as `container.id` and `container.log.tag` is dropped because it is already present as `log.syslog.appname`. The field `container.partial` is replaced by the tag `partial_message` if it was `true`, otherwise no tag is added. --- CHANGELOG.next.asciidoc | 7 +++ filebeat/docs/inputs/input-journald.asciidoc | 14 ++--- filebeat/input/journald/input.go | 15 ++++++ .../input/journald/input_filtering_test.go | 4 +- filebeat/input/journald/input_parsers_test.go | 30 +++++++++++ filebeat/input/journald/input_test.go | 6 +++ .../journald/pkg/journalfield/conv_test.go | 21 ++++---- .../journald/pkg/journalfield/default.go | 12 ++--- .../input-multiline-parser-events.json | 50 ++++-------------- .../journald/testdata/ndjson-parser.export | Bin 830 -> 861 bytes .../testdata/ndjson-parser.journal.gz | Bin 10227 -> 10320 bytes .../auth/test/debian-12.journal-expected.json | 16 ++++++ .../test/debian-12.journal-expected.json | 3 ++ 13 files changed, 113 insertions(+), 65 deletions(-) diff --git a/CHANGELOG.next.asciidoc b/CHANGELOG.next.asciidoc index 91e75266659d..96576690f1e4 100644 --- a/CHANGELOG.next.asciidoc +++ b/CHANGELOG.next.asciidoc @@ -59,6 +59,13 @@ https://github.com/elastic/beats/compare/v8.8.1\...main[Check the HEAD diff] - Filestream inputs can define `allow_deprecated_id_duplication: true` to run keep the previous behaviour of running inputs with duplicated IDs. {issue}41938[41938] {pull}41954[41954] - The Filestream input only starts to ingest a file when it is >= 1024 bytes in size. This happens because the fingerprint` is the default file identity now. To restore the previous behaviour, set `file_identity.native: ~` and `prospector.scanner.fingerprint.enabled: false` {issue}40197[40197] {pull}41762[41762] - Filebeat fails to start when its configuration contains usage of the deprecated `log` or `container` inputs. However, they can still be using while setting `allow_deprecated_use: true` in their configuration {pull}42295[42295] +- The fields produced by the Journald input are updated to better match ECS. Renamed fields: +Dropped fields: `syslog.priority` and `syslog.facility` while keeping their duplicated equivalent: +`log.syslog.priority`,`log.syslog.facility.code`. Renamed fields: `syslog.identifier` -> `log.syslog.appname`, +`syslog.pid` -> `log.syslog.procid`. `container.id_truncated` is dropped because the full container ID is +already present as `container.id` and `container.log.tag` is dropped because it is already present as +`log.syslog.appname`. The field `container.partial` is replaced by the tag `partial_message` if it was `true`, +otherwise no tag is added. {issue}42208[42208] {pull}42403[42403] *Heartbeat* diff --git a/filebeat/docs/inputs/input-journald.asciidoc b/filebeat/docs/inputs/input-journald.asciidoc index f7655f51419f..d2bf341d60f0 100644 --- a/filebeat/docs/inputs/input-journald.asciidoc +++ b/filebeat/docs/inputs/input-journald.asciidoc @@ -461,10 +461,10 @@ journald fields: `_MACHINE_ID`:: `host.id` `_MESSAGE`:: `message` `_PID`:: `process.pid` -`_PRIORITY`:: `syslog.priority` -`_SYSLOG_FACILITY`:: `syslog.facility` -`_SYSLOG_IDENTIFIER`:: `syslog.identifier` -`_SYSLOG_PID`:: `syslog.pid` +`_PRIORITY`:: `log.syslog.priority` +`_SYSLOG_FACILITY`:: `log.syslog.facility.code` +`_SYSLOG_IDENTIFIER`:: `log.syslog.appname` +`_SYSLOG_PID`:: `log.syslog.procid` `_SYSTEMD_CGROUP`:: `systemd.cgroup` `_SYSTEMD_INVOCATION_ID`:: `systemd.invocation_id` `_SYSTEMD_OWNER_UID`:: `systemd.owner_uid` @@ -484,13 +484,13 @@ https://docs.docker.com/config/containers/logging/journald/[Docker] are also available: [horizontal] -`CONTAINER_ID`:: `container.id_truncated` `CONTAINER_ID_FULL`:: `container.id` `CONTAINER_NAME`:: `container.name` -`CONTAINER_PARTIAL_MESSAGE`:: `container.partial` -`CONTAINER_TAG`:: `container.log.tag` `IMAGE_NAME`:: `container.image.name` +If `CONTAINER_PARTIAL_MESSAGE` is present and it is true, then the tag +`partial_message` is added to the final event. + [id="{beatname_lc}-input-{type}-common-options"] include::../inputs/input-common-options.asciidoc[] diff --git a/filebeat/input/journald/input.go b/filebeat/input/journald/input.go index 0ab3c5481775..e8eb8a7a8773 100644 --- a/filebeat/input/journald/input.go +++ b/filebeat/input/journald/input.go @@ -22,6 +22,7 @@ package journald import ( "errors" "fmt" + "strconv" "time" "github.com/elastic/beats/v7/filebeat/input/journald/pkg/journalctl" @@ -279,6 +280,20 @@ func (r *readerAdapter) Next() (reader.Message, error) { fields.Put("event.kind", "event") fields.Put("event.created", created) + // IF 'container.partial' is present, we can parse it and it's true, then + // add 'partial_message' to tags. + if partialMessageRaw, err := fields.GetValue("container.partial"); err == nil { + partialMessage, err := strconv.ParseBool(fmt.Sprint(partialMessageRaw)) + if err == nil && partialMessage { + // 'fields' came directly from the journal, + // so there is no chance tags already exist + fields.Put("tags", []string{"partial_message"}) + } + } + + // Delete 'container.partial', if there are any errors, ignore it + _ = fields.Delete("container.partial") + // if entry is coming from a remote journal, add_host_metadata overwrites // the source hostname, so it has to be copied to a different field if r.saveRemoteHostname { diff --git a/filebeat/input/journald/input_filtering_test.go b/filebeat/input/journald/input_filtering_test.go index 9464016d40dd..34d0755393bd 100644 --- a/filebeat/input/journald/input_filtering_test.go +++ b/filebeat/input/journald/input_filtering_test.go @@ -142,7 +142,7 @@ func TestInputIncludeMatches(t *testing.T) { "single match condition": { includeMatches: map[string]interface{}{ "match": []string{ - "syslog.facility=3", + "log.syslog.facility.code=3", }, }, expectedMessages: []string{ @@ -159,7 +159,7 @@ func TestInputIncludeMatches(t *testing.T) { includeMatches: map[string]interface{}{ "match": []string{ "journald.process.name=systemd", - "syslog.facility=3", + "log.syslog.facility.code=3", }, }, expectedMessages: []string{ diff --git a/filebeat/input/journald/input_parsers_test.go b/filebeat/input/journald/input_parsers_test.go index 35df3cc1e848..de9becc386f2 100644 --- a/filebeat/input/journald/input_parsers_test.go +++ b/filebeat/input/journald/input_parsers_test.go @@ -31,6 +31,9 @@ import ( // it only tests a single parser, but that is enough to ensure // we're correctly using the parsers func TestInputParsers(t *testing.T) { + // If this test fails, uncomment the lopg setup line + // to send logs to stderr + // logp.DevelopmentSetup() out := decompress(t, filepath.Join("testdata", "ndjson-parser.journal.gz")) env := newInputTestingEnvironment(t) @@ -71,3 +74,30 @@ func TestInputParsers(t *testing.T) { t.Errorf("expecting 'answer' from the Journal JSON to be '%d' got '%d' instead", expectedAnswer, answer) } } + +func TestPartialMessageTag(t *testing.T) { + out := decompress(t, filepath.Join("testdata", "ndjson-parser.journal.gz")) + env := newInputTestingEnvironment(t) + inp := env.mustCreateInput(mapstr.M{ + "paths": []string{out}, + }) + + ctx, cancelInput := context.WithCancel(context.Background()) + t.Cleanup(cancelInput) + env.startInput(ctx, inp) + env.waitUntilEventCount(1) + event := env.pipeline.clients[0].GetEvents()[0] + + tags, err := event.Fields.GetValue("tags") + if err != nil { + t.Fatalf("'tags' not found in event: %s", err) + } + + tagsStrSlice, ok := tags.([]string) + if !ok { + t.Fatalf("expecting 'tags' to be []string, got %T instead", tags) + } + if tagsStrSlice[0] != "partial_message" { + t.Fatalf("expecting the tag 'partial_message', got %v instead", tagsStrSlice) + } +} diff --git a/filebeat/input/journald/input_test.go b/filebeat/input/journald/input_test.go index 776115d5d8ac..5d0edb18cb05 100644 --- a/filebeat/input/journald/input_test.go +++ b/filebeat/input/journald/input_test.go @@ -119,6 +119,12 @@ func TestInputFieldsTranslation(t *testing.T) { // events in the same format as the original one. We use the events from the // already existing journal file 'input-multiline-parser.journal' // +// Generating golden file: to generate the golden file you need to copy +// and run this test on a older version that still uses go-systemd, +// like 8.16.0, so the input run on this older version, call +// `env.pipeline.GetAllEvents()`, get the events, marshal them as +// JSON with " " as the indent argument and write it to the file. +// // The following fields are not currently tested: // __CURSOR - it is added to the registry and there are other tests for it // __MONOTONIC_TIMESTAMP - it is part of the cursor diff --git a/filebeat/input/journald/pkg/journalfield/conv_test.go b/filebeat/input/journald/pkg/journalfield/conv_test.go index c92d3f15bcb0..936b7ce485ee 100644 --- a/filebeat/input/journald/pkg/journalfield/conv_test.go +++ b/filebeat/input/journald/pkg/journalfield/conv_test.go @@ -50,8 +50,10 @@ func TestConversion(t *testing.T) { "SYSLOG_PID": "123456", }, want: mapstr.M{ - "syslog": mapstr.M{ - "pid": int64(123456), + "log": mapstr.M{ + "syslog": mapstr.M{ + "procid": int64(123456), + }, }, }, }, @@ -60,9 +62,6 @@ func TestConversion(t *testing.T) { "PRIORITY": "123456, ", }, want: mapstr.M{ - "syslog": mapstr.M{ - "priority": int64(123456), - }, "log": mapstr.M{ "syslog": mapstr.M{ "priority": int64(123456), @@ -75,8 +74,10 @@ func TestConversion(t *testing.T) { "SYSLOG_PID": "123456,root", }, want: mapstr.M{ - "syslog": mapstr.M{ - "pid": int64(123456), + "log": mapstr.M{ + "syslog": mapstr.M{ + "procid": int64(123456), + }, }, }, }, @@ -85,8 +86,10 @@ func TestConversion(t *testing.T) { "SYSLOG_PID": "", }, want: mapstr.M{ - "syslog": mapstr.M{ - "pid": "", + "log": mapstr.M{ + "syslog": mapstr.M{ + "procid": "", + }, }, }, }, diff --git a/filebeat/input/journald/pkg/journalfield/default.go b/filebeat/input/journald/pkg/journalfield/default.go index d2ec76fcd393..bc508750765b 100644 --- a/filebeat/input/journald/pkg/journalfield/default.go +++ b/filebeat/input/journald/pkg/journalfield/default.go @@ -38,10 +38,10 @@ var journaldEventFields = FieldConversion{ "OBJECT_SYSTEMD_UNIT": text("journald.object.systemd.unit"), "OBJECT_SYSTEMD_USER_UNIT": text("journald.object.systemd.user_unit"), "OBJECT_UID": integer("journald.object.uid"), - "PRIORITY": integer("syslog.priority", "log.syslog.priority"), - "SYSLOG_FACILITY": integer("syslog.facility", "log.syslog.facility.code"), - "SYSLOG_IDENTIFIER": text("syslog.identifier"), - "SYSLOG_PID": integer("syslog.pid"), + "PRIORITY": integer("log.syslog.priority"), + "SYSLOG_FACILITY": integer("log.syslog.facility.code"), + "SYSLOG_IDENTIFIER": text("log.syslog.appname"), + "SYSLOG_PID": integer("log.syslog.procid"), "UNIT": text("journald.unit"), "_AUDIT_LOGINUID": integer("journald.audit.login_uid"), "_AUDIT_SESSION": text("journald.audit.session"), @@ -74,14 +74,14 @@ var journaldEventFields = FieldConversion{ "_UID": integer("journald.uid"), // docker journald fields from: https://docs.docker.com/config/containers/logging/journald/ - "CONTAINER_ID": text("container.id_truncated"), "CONTAINER_ID_FULL": text("container.id"), "CONTAINER_NAME": text("container.name"), - "CONTAINER_TAG": text("container.log.tag"), "CONTAINER_PARTIAL_MESSAGE": text("container.partial"), "IMAGE_NAME": text("container.image.name"), // dropped fields + "CONTAINER_ID": ignoredField, + "CONTAINER_TAG": ignoredField, "_SOURCE_MONOTONIC_TIMESTAMP": ignoredField, // saved in the registry "_SOURCE_REALTIME_TIMESTAMP": ignoredField, // saved in the registry "__CURSOR": ignoredField, // saved in the registry diff --git a/filebeat/input/journald/testdata/input-multiline-parser-events.json b/filebeat/input/journald/testdata/input-multiline-parser-events.json index 0d8fbff7f58d..166d97bfa3a5 100644 --- a/filebeat/input/journald/testdata/input-multiline-parser-events.json +++ b/filebeat/input/journald/testdata/input-multiline-parser-events.json @@ -33,6 +33,7 @@ }, "log": { "syslog": { + "appname": "sudo", "facility": { "code": 10 }, @@ -98,11 +99,6 @@ } } }, - "syslog": { - "facility": 10, - "identifier": "sudo", - "priority": 6 - }, "systemd": { "cgroup": "/user.slice/user-1000.slice/session-1.scope", "invocation_id": "2172b101bd684f7b9ca1c434ff2cdd80", @@ -165,8 +161,10 @@ }, "uid": 1000 }, + "message_id": "39f53479d3a045ac8e11786248231fbf", "log": { "syslog": { + "appname": "systemd", "facility": { "code": 3 }, @@ -174,7 +172,6 @@ } }, "message": "Started Outputs some log lines.", - "message_id": "39f53479d3a045ac8e11786248231fbf", "process": { "args": [ "/usr/lib/systemd/systemd", @@ -184,11 +181,6 @@ "command_line": "/usr/lib/systemd/systemd --user", "pid": 1003 }, - "syslog": { - "facility": 3, - "identifier": "systemd", - "priority": 6 - }, "systemd": { "cgroup": "/user.slice/user-1000.slice/user@1000.service/init.scope", "owner_uid": 1000, @@ -240,6 +232,7 @@ }, "log": { "syslog": { + "appname": "cat", "facility": { "code": 3 }, @@ -250,11 +243,6 @@ "process": { "pid": 2084785 }, - "syslog": { - "facility": 3, - "identifier": "cat", - "priority": 6 - }, "systemd": { "cgroup": "/user.slice/user-1000.slice/user@1000.service/app.slice/log-service.service", "invocation_id": "ceb98539accb4f60be76734b4ddd6d44", @@ -307,6 +295,7 @@ }, "log": { "syslog": { + "appname": "cat", "facility": { "code": 3 }, @@ -317,11 +306,6 @@ "process": { "pid": 2084785 }, - "syslog": { - "facility": 3, - "identifier": "cat", - "priority": 6 - }, "systemd": { "cgroup": "/user.slice/user-1000.slice/user@1000.service/app.slice/log-service.service", "invocation_id": "ceb98539accb4f60be76734b4ddd6d44", @@ -374,6 +358,7 @@ }, "log": { "syslog": { + "appname": "cat", "facility": { "code": 3 }, @@ -384,11 +369,6 @@ "process": { "pid": 2084785 }, - "syslog": { - "facility": 3, - "identifier": "cat", - "priority": 6 - }, "systemd": { "cgroup": "/user.slice/user-1000.slice/user@1000.service/app.slice/log-service.service", "invocation_id": "ceb98539accb4f60be76734b4ddd6d44", @@ -441,6 +421,7 @@ }, "log": { "syslog": { + "appname": "cat", "facility": { "code": 3 }, @@ -451,11 +432,6 @@ "process": { "pid": 2084785 }, - "syslog": { - "facility": 3, - "identifier": "cat", - "priority": 6 - }, "systemd": { "cgroup": "/user.slice/user-1000.slice/user@1000.service/app.slice/log-service.service", "invocation_id": "ceb98539accb4f60be76734b4ddd6d44", @@ -508,6 +484,7 @@ }, "log": { "syslog": { + "appname": "cat", "facility": { "code": 3 }, @@ -518,11 +495,6 @@ "process": { "pid": 2084785 }, - "syslog": { - "facility": 3, - "identifier": "cat", - "priority": 6 - }, "systemd": { "cgroup": "/user.slice/user-1000.slice/user@1000.service/app.slice/log-service.service", "invocation_id": "ceb98539accb4f60be76734b4ddd6d44", @@ -575,6 +547,7 @@ }, "log": { "syslog": { + "appname": "cat", "facility": { "code": 3 }, @@ -585,11 +558,6 @@ "process": { "pid": 2084785 }, - "syslog": { - "facility": 3, - "identifier": "cat", - "priority": 6 - }, "systemd": { "cgroup": "/user.slice/user-1000.slice/user@1000.service/app.slice/log-service.service", "invocation_id": "ceb98539accb4f60be76734b4ddd6d44", diff --git a/filebeat/input/journald/testdata/ndjson-parser.export b/filebeat/input/journald/testdata/ndjson-parser.export index 0a24b593f776c6d9855902cdb759b7e578756830..a85e04ceb86bfeeda42f83e91dacafb033cf82ac 100644 GIT binary patch delta 61 zcmdnTc9(5~2D7QNzh8)>r=M$3e1Kz6h^M1Zysv9;u%o-HZAnpSDp!1fr;DwjnW32( QSG=>oudi)#VhI-)07lml%>V!Z delta 30 lcmcc1wvTOt2D4avfTxSCp_!qX8CSfszpt-ta$*S=7XXRu2s8iy diff --git a/filebeat/input/journald/testdata/ndjson-parser.journal.gz b/filebeat/input/journald/testdata/ndjson-parser.journal.gz index f5eb86364da58dd9447790e74fd3f040dea6fa2d..a279f6238bca2b381538fa9339d5d101287779ce 100644 GIT binary patch literal 10320 zcmeI1X;72b8ppX^L~ARx(4qxRU08$yDk?&N+_6r*z!j=sp+Jbprfd-cM3y9WL97vS z4I*0#BM32I2#A7&Kq_g2!331;We+KV2nh)gLP*{$*UsEeUOw3A+!^f6`|X@L&w2mP z@0|00&O0vzzj0&F&xJ8YZzo2^qL7IW7sHcLQOS<6$kgP-@OUH15GFo!pxF2lYBi%R$V(O;Ee;bB&Fzqc|q z$W?tysbSIWIZC1Wv_$Yk1q*6&jt|B}f^>r>yP;yx;jni7bbFytKJ?xHbZ2PvDfNzPoti(b z-7vvlm;l!bYVzM^_@9K&CJqHykYDF3;xZ5o?pxCn|HACvfYc@<%^a@!Z!+2s3|P0g za$sZiFE|#Z$kooS9ymHytubXQ5COJkpxl1+X>soNRksFj{WCN%;Q}vm!-j)N)l|wp z|D9L6a^I$ML?Wx4r{;6Qd`p$m2D@sagl`*3SBU#u(i2KPwP0n1>rAZD zg;jiqM{^;@<(sCOY~+5qrEnWAZZ?707BkP{t43=?2nAWhq-KpW@LN1Qw3xZh%Gsnw zaf*$K?}5&xg~C$&2ju9;RuE;5Ac<{ffSC~smWI=F$~i;$q?ex+D`L;eATuu~rl8Y_ zXqu~n5c4IPL7wx>K*cS=F=93JJ@~I<^mE|hkPSIa@neiChJY50D758l-U3^Ab}}a| z`$*g;JM!|rvFUfRZke3u!!BrO6HCQ4VAl$jYYqZtU!VN?Cxw4Dc7&LPU{&HbM+7ns zU*~UEdlAa(GQS}PW}Va6*Lsr+Yni@~S%(?7z!h^xahHNS)^@vGPRtU52siQVlf}sF zMf<$to!iUnn4Y2J6WC|C&OGpomN{?^o-cdy5aAp;k=7j$@$?F9#6762B}`xps+@Gw zWF;bL^2S#&h$x&BP)yN3-wXIMS{2L*my11{lmh@%l}b2*_B~${T}_kcOVo<(J~ApO zx&Kv+*ys7;+?|;gJB|}=$gN5FZ8x+=N^>4FD&s+ z_`dj7Zf;d_1gg5>P%D3CuiPAt(eV>95^;A!U|Y`&p5y`f!Wr0@{KsaE_Ke_2mcld8 zJOd9O?hXaJmVTb;y;aS9&dkAB~VzX?@4(y-zNk$6DC%4 znM|uKl5rKvG&4GCo&}f?Zqn+~r!i?yhn%UcgKaqEHYVNLlf7C@53dYa9++_A!o-|o z16s4p`|!5$?6Py!%WnMU2nyZm;P{+H!czPY-1XL+54diY@`r?(V_ z6CWSYTEfv8sJ1rMvA|5Tx)ak1PMfHi>8UR??E^6W30c<{ol{wlr{s`Y<{dGAr+a8B zOrV+#8t{K1PK2RjvyZAR%R(B0NYK!oJ15*9Oq0&5?h78@rcMuIauiJ$Rn_K*Rlo-O z>5S_(wQrBJ2M$SW=4ewRGs^=BF1Xblsz!qny6cj zDpfnFrJIy}KtVZUAS|is7((EV_7dv|^BLI5%-h}9REzdvZVfJhSPrsP&7ULNm!+ms zd;Y_wXEz*tu>%w-$?<#+X3iT<8Dh={C~SfpH*#OJzA6)Lzv289;rOah21asWE6&k#~3Sb=|RGiCc7(-2cRn(s;iES_tId@+@PbzJ_TThs~2{O(mJA>D)m9WKegqGip)MaqY=tX2CK!M z%`PZ43onkXXt+Ulw$^Kx1yH6Gs_i5LbFtEBo~(I2uaq_a*$Npk?u$e=}_$>S#-EsNk#>v z4_A2gemNL>Nq#tNBNz}wV5uR4kLz_;QmyZ03S75{y)Gqope&@L3ydE?tFc0XauL!*&t~k5a8$Zh}aDc z8pOz!4AQd177&CWghWN55!s|eL?97a0)!CALdZrY?Nm+m`|DP{bH4j-y;tuHhsr_# zoBSO+o>fF8Y&FJROCsWN&dKov;&p;c5 z?7Fef6KchpM+f1a!>==+`CPwqBeB!%5EWZx?QaA+uC2y;I;iwqke%N51ph(lwMbR)Ng!(__hRY4 z^@M-SegB=+c1PH0xdcHO3UUZ^bhVngWy=GHuQxhdW z3FKhPF~OS7ah&>qe-z% z!LtSS?BNbPL^yJG=1X{U!qh`K5{gUXhxu`6a_)XdRuTF2hAq`ctTQf-Hx_~}FGnHv z*6u5wHgW^qe1;u)@b?l!9SdS{oHCEX?ONiE?TtVZ=LBkR{Wimf9q6X@Q=0i#gA3?$ zS5kIH1-vDGdyA3f&8gNai2Lv@e_e-8C!+7eTdKZ+Zw1{9bEm#FK;n(-cYM}~?*?5s zW@Nzo8G(9!K4~ z>45re29AZ11&QfAap>{B|9fvvwP5`3!wsfr!<={Q-+Ue6==OD1q2U4y{!&mAwCwo_ zxSc@~2wMK7@X4L8Jsftt)bo5|s=uBo2*es4!xV=7+1v-{epg&UpOpb|n)3I3di9kx z;iZV4>OzqxyURx~O7QbP;?~H4#*`1d$^C@E+K=+6Ke;K~-AnyY;dLudf_z~veM`1B zhnWOHgiF6^1AiY@LmsiZz`#8e&e{SUS_-5F{D(HOUL;z;BuX%SKz$WXu0lzC7an() z6=eda3d)}!f&XHX8cmr@?rpz8UrwC~WW%QXl{GaT9UTkmTF489RY~BGS`B(|E2W>J z%Ig(JLf+F$-IX6+F@B~|gs?@i5Vodz@U6uVlKAp>1-{_Bp2sU!DQF8^)Vd-DPh!lf z$}PP!2@u*t^gCun8M?-vI5L!IJ=ImxeBDwADASo=)&PZZXy_M%zpMZQWCD!dysQkUwd z%c5u+%E#O1o1Va;q#c%>AvgMz?{Toix}`m+i;*GO6g>+k+OL`9y1N#jfW2eIU|3js zFvTn!qnaq;0qB%Pmqk zq_y;M&Rlsd$ni*}^0@Auj~#eTr2izpDwxh4IP8JX0pc~fT|-<(K$)X`gSS*2nh><6 zZsU%}>HE*61(T;kfQz*PR7<~TX~nW*5^4cG>XeixStFhUd(R=zdfk`hJip4bF^?*_ z=LU#zlssPChO(?EjtTUM&{#|oF0tPfKhj04u4CFWewk5wl-Ik;$K+$mgk~9oY{G3h z$?fuPNWVnd#Icu_v%xVZh}VFRmIrx31H95%oi#)NY!oR1S6nhG2gFEKThCZ*_s(3l zs>>wl^4DeONwjE~;!DIoB|H!5WS*j+AcSWsD$RI=>pU%h*_E;qq`VcK|rNfNHNh{waqn38bq=tT|-WtK&oJrWN^_qq1~6 z&y>+VN7qq*iwZS*0d{t-nio42AM zuE1CSq6!vQp$iSqnMiru=q)HkW31!QI@V@8+q{>kQ9r_9gHWipyH6Vi#*~*L5@s;5 z&?QZl?kLnb6AO$$z~3$w_4J-i9hK!;S@KfTT{2eNx&x63qT!iQQdq_e)eWFaqyvAmwdCetF^U? z!z6Jk_bz2`(nG+~$~^p@883?0M33q9vm{jiXeuhQ#B;+nApF7E|HY;fQI6{iA=^i^ zCVjeBoWO5Mn``B4i1*X8&JZte_#DWg<_aLw>7g54<~h_`n?Bm$9l9SDCj4!z)WXQ& zfZJ?$c}f+2cQ#^y=f~Nn{&m>w7TdO>cUo&EjQ0S#GY$-LlEJrRkmZ(my_qKVV?_7n zzV+7C&D%O#md&(}>Cjb@&1>0STK8&puJBRw4Z+m7%*c>Y3FgJCMhDIGXtO;#|ICnW ZIzVh{Y={=H(XM23sT+M@%a)s4{tNd`$0h&( diff --git a/filebeat/module/system/auth/test/debian-12.journal-expected.json b/filebeat/module/system/auth/test/debian-12.journal-expected.json index 2ef69b76b22a..21648f491b58 100644 --- a/filebeat/module/system/auth/test/debian-12.journal-expected.json +++ b/filebeat/module/system/auth/test/debian-12.journal-expected.json @@ -17,8 +17,10 @@ "host.hostname": "vagrant-debian-12", "host.id": "5e6dc8fe417f4ea383e2afaa731f5d8a", "input.type": "journald", + "log.syslog.appname": "sshd", "log.syslog.facility.code": 4, "log.syslog.priority": 6, + "log.syslog.procid": 26538, "message": "Accepted publickey for vagrant from 10.0.2.2 port 48274 ssh2: ED25519 SHA256:k1kjhwoH/H3w31MbGOIGd7qxrkSQJnoAN0eYJVHDmmI", "process.args": [ "\"sshd: vagrant [priv]\"" @@ -65,8 +67,10 @@ "host.hostname": "vagrant-debian-12", "host.id": "5e6dc8fe417f4ea383e2afaa731f5d8a", "input.type": "journald", + "log.syslog.appname": "sshd", "log.syslog.facility.code": 4, "log.syslog.priority": 6, + "log.syslog.procid": 1710, "message": "Accepted password for vagrant from 192.168.42.119 port 55310 ssh2", "process.args": [ "\"sshd: vagrant [priv]\"" @@ -111,8 +115,10 @@ "host.hostname": "vagrant-debian-12", "host.id": "5e6dc8fe417f4ea383e2afaa731f5d8a", "input.type": "journald", + "log.syslog.appname": "sshd", "log.syslog.facility.code": 4, "log.syslog.priority": 6, + "log.syslog.procid": 1721, "message": "Invalid user test from 192.168.42.119 port 48890", "process.args": [ "\"sshd: unknown [priv]\"" @@ -155,8 +161,10 @@ "host.hostname": "vagrant-debian-12", "host.id": "5e6dc8fe417f4ea383e2afaa731f5d8a", "input.type": "journald", + "log.syslog.appname": "sshd", "log.syslog.facility.code": 4, "log.syslog.priority": 6, + "log.syslog.procid": 1723, "message": "Failed password for root from 192.168.42.119 port 46632 ssh2", "process.args": [ "\"sshd: root [priv]\"" @@ -201,8 +209,10 @@ "host.hostname": "vagrant-debian-12", "host.id": "5e6dc8fe417f4ea383e2afaa731f5d8a", "input.type": "journald", + "log.syslog.appname": "sshd", "log.syslog.facility.code": 4, "log.syslog.priority": 6, + "log.syslog.procid": 1723, "message": "Failed password for root from 192.168.42.119 port 46632 ssh2", "process.args": [ "\"sshd: root [priv]\"" @@ -247,8 +257,10 @@ "host.hostname": "vagrant-debian-12", "host.id": "5e6dc8fe417f4ea383e2afaa731f5d8a", "input.type": "journald", + "log.syslog.appname": "sshd", "log.syslog.facility.code": 4, "log.syslog.priority": 6, + "log.syslog.procid": 1723, "message": "Failed password for root from 192.168.42.119 port 46632 ssh2", "process.args": [ "\"sshd: root [priv]\"" @@ -285,6 +297,7 @@ "host.hostname": "vagrant-debian-12", "host.id": "5e6dc8fe417f4ea383e2afaa731f5d8a", "input.type": "journald", + "log.syslog.appname": "sudo", "log.syslog.facility.code": 10, "log.syslog.priority": 5, "message": " vagrant : TTY=pts/2 ; PWD=/home/vagrant ; USER=root ; COMMAND=/usr/bin/emacs /etc/ssh/sshd_config", @@ -333,8 +346,10 @@ "host.hostname": "vagrant-debian-12", "host.id": "5e6dc8fe417f4ea383e2afaa731f5d8a", "input.type": "journald", + "log.syslog.appname": "groupadd", "log.syslog.facility.code": 10, "log.syslog.priority": 6, + "log.syslog.procid": 1743, "message": "new group: name=test, GID=1001", "process.args": [ "/sbin/groupadd", @@ -363,6 +378,7 @@ "host.hostname": "vagrant-debian-12", "host.id": "5e6dc8fe417f4ea383e2afaa731f5d8a", "input.type": "journald", + "log.syslog.appname": "systemd-logind", "log.syslog.facility.code": 4, "log.syslog.priority": 6, "message": "Session 8 logged out. Waiting for processes to exit.", diff --git a/filebeat/module/system/syslog/test/debian-12.journal-expected.json b/filebeat/module/system/syslog/test/debian-12.journal-expected.json index b75cce10fc8a..bdfbdb0d7975 100644 --- a/filebeat/module/system/syslog/test/debian-12.journal-expected.json +++ b/filebeat/module/system/syslog/test/debian-12.journal-expected.json @@ -8,6 +8,7 @@ "host.hostname": "vagrant-debian-12", "host.id": "5e6dc8fe417f4ea383e2afaa731f5d8a", "input.type": "journald", + "log.syslog.appname": "systemd", "log.syslog.facility.code": 3, "log.syslog.priority": 6, "message": "Stopped target getty.target - Login Prompts.", @@ -34,6 +35,7 @@ "host.hostname": "vagrant-debian-12", "host.id": "5e6dc8fe417f4ea383e2afaa731f5d8a", "input.type": "journald", + "log.syslog.appname": "kernel", "log.syslog.facility.code": 0, "log.syslog.priority": 6, "message": "Console: switching to colour frame buffer device 160x50", @@ -51,6 +53,7 @@ "host.hostname": "bookworm", "host.id": "5e6dc8fe417f4ea383e2afaa731f5d8a", "input.type": "journald", + "log.syslog.appname": "kernel", "log.syslog.facility.code": 0, "log.syslog.priority": 6, "message": "thermal_sys: Registered thermal governor 'power_allocator'", From 0e9fd0f6a33fda5a1c203666bd795129c7b3842c Mon Sep 17 00:00:00 2001 From: Victor Martinez Date: Fri, 31 Jan 2025 06:46:34 +0100 Subject: [PATCH 2/5] mergify: remove backport-8.x enforcement (#42510) --- .mergify.yml | 14 -------------- 1 file changed, 14 deletions(-) diff --git a/.mergify.yml b/.mergify.yml index aa850556a5d1..ea8948e485b0 100644 --- a/.mergify.yml +++ b/.mergify.yml @@ -147,20 +147,6 @@ pull_request_rules: branches, such as: * `backport-8./d` is the label to automatically backport to the `8./d` branch. `/d` is the digit - - name: add backport-8.x label for main only if no skipped or assigned already - conditions: - - -label~=^(backport-skip|backport-8.x)$ - - base=main - - -merged - - -closed - actions: - comment: - message: | - `backport-8.x` has been added to help with the transition to the new branch `8.x`. - If you don't need it please use `backport-skip` label and remove the `backport-8.x` label. - label: - add: - - backport-8.x - name: notify the backport has not been merged yet conditions: - -merged From 8f25b1dfaab026f2b9036c304c18853122f5b731 Mon Sep 17 00:00:00 2001 From: "mergify[bot]" <37929162+mergify[bot]@users.noreply.github.com> Date: Fri, 31 Jan 2025 08:37:30 +0100 Subject: [PATCH 3/5] Use centralized version qualifier (#42503) (#42506) * Use centralized version qualifier To avoid manual invocations specifying the version qualifier for prereleases in this commit we leverage a centralized version of truth for the version qualifier. * fix bug * revert defaults (cherry picked from commit ad052340f1f4aab382971eb27d8eade688444022) Co-authored-by: Dimitrios Liappis --- .buildkite/packaging.pipeline.yml | 26 ++++++++++++++++--------- .buildkite/scripts/version_qualifier.sh | 22 +++++++++++++++++++++ 2 files changed, 39 insertions(+), 9 deletions(-) create mode 100755 .buildkite/scripts/version_qualifier.sh diff --git a/.buildkite/packaging.pipeline.yml b/.buildkite/packaging.pipeline.yml index 2f3ff489e4ac..34b103b23736 100644 --- a/.buildkite/packaging.pipeline.yml +++ b/.buildkite/packaging.pipeline.yml @@ -61,7 +61,7 @@ steps: - label: Staging dashboards # TODO remove OR clause below (see earlier comment) - if: build.branch =~ /^[0-9]+\.[0-9x]+\$/ || build.env('VERSION_QUALIFIER') != null + if: (build.branch =~ /^[0-9]+\.[0-9x]+\$/ || build.env('VERSION_QUALIFIER') != null) || build.env('RUN_STAGING') == "true" depends_on: start-gate-staging key: dashboards-staging # TODO: container with go and make @@ -73,9 +73,10 @@ steps: retry: automatic: - limit: 1 - commands: - - make build/distributions/dependencies.csv - - make beats-dashboards + commands: | + source .buildkite/scripts/version_qualifier.sh + make build/distributions/dependencies.csv + make beats-dashboards env: SNAPSHOT: false DEV: false @@ -174,14 +175,16 @@ steps: key: packaging-staging depends_on: start-gate-staging # TODO remove OR clause below (see earlier comment) - if: build.branch =~ /^[0-9]+\.[0-9x]+\$/ || build.env('VERSION_QUALIFIER') != null + if: (build.branch =~ /^[0-9]+\.[0-9x]+\$/ || build.env('VERSION_QUALIFIER') != null) || build.env('RUN_STAGING') == "true" steps: - label: "STAGING: {{matrix}}" env: PLATFORMS: "${PLATFORMS}" SNAPSHOT: false DEV: false - command: ".buildkite/scripts/packaging/package-dra.sh {{matrix}}" + command: | + source .buildkite/scripts/version_qualifier.sh + .buildkite/scripts/packaging/package-dra.sh {{matrix}} agents: provider: gcp image: "${IMAGE_UBUNTU_X86_64}" @@ -214,7 +217,9 @@ steps: PACKAGES: "docker" SNAPSHOT: false DEV: false - command: ".buildkite/scripts/packaging/package-dra.sh {{matrix}}" + command: | + source .buildkite/scripts/version_qualifier.sh + .buildkite/scripts/packaging/package-dra.sh {{matrix}} agents: provider: "aws" imagePrefix: "${AWS_IMAGE_UBUNTU_ARM_64}" @@ -244,7 +249,9 @@ steps: PLATFORMS: "${PLATFORMS}" SNAPSHOT: false DEV: false - command: ".buildkite/scripts/packaging/package-dra.sh x-pack/agentbeat" + command: | + source .buildkite/scripts/version_qualifier.sh + .buildkite/scripts/packaging/package-dra.sh x-pack/agentbeat agents: provider: gcp image: "${IMAGE_UBUNTU_X86_64}" @@ -280,7 +287,7 @@ steps: - label: DRA Staging # TODO remove OR clause below (see earlier comment) - if: build.branch =~ /^[0-9]+\.[0-9x]+\$/ || build.env('VERSION_QUALIFIER') != null + if: (build.branch =~ /^[0-9]+\.[0-9x]+\$/ || build.env('VERSION_QUALIFIER') != null) || build.env('RUN_STAGING') == "true" key: dra-staging env: DRA_WORKFLOW: staging @@ -289,6 +296,7 @@ steps: - packaging-staging - dashboards-staging command: | + source .buildkite/scripts/version_qualifier.sh buildkite-agent artifact download "build/**" . .buildkite/scripts/packaging/prepare-release-manager.sh staging .buildkite/scripts/dra.sh diff --git a/.buildkite/scripts/version_qualifier.sh b/.buildkite/scripts/version_qualifier.sh new file mode 100755 index 000000000000..88d172dad588 --- /dev/null +++ b/.buildkite/scripts/version_qualifier.sh @@ -0,0 +1,22 @@ +#!/usr/bin/env bash + +# An opinionated approach to managing the Elastic Qualifier for the DRA in a Google Bucket +# instead of using a Buildkite env variable. + +if [[ -n "$VERSION_QUALIFIER" ]]; then + echo "~~~ VERSION_QUALIFIER externally set to [$VERSION_QUALIFIER]" + return 0 +fi + +# DRA_BRANCH can be used for manually testing packaging with PRs +# e.g. define `DRA_BRANCH="main"` under Options/Environment Variables in the Buildkite UI after clicking new Build +BRANCH="${DRA_BRANCH:="${BUILDKITE_BRANCH:=""}"}" + +qualifier="" +URL="https://storage.googleapis.com/dra-qualifier/${BRANCH}" +if curl -sf -o /dev/null "$URL" ; then + qualifier=$(curl -s "$URL") +fi + +export VERSION_QUALIFIER="$qualifier" +echo "~~~ VERSION_QUALIFIER set to [$VERSION_QUALIFIER]" From 174c08dfe70b1fc3c1cc74d4f44ee1ebaa82716f Mon Sep 17 00:00:00 2001 From: Dimitrios Liappis Date: Fri, 31 Jan 2025 09:44:16 +0200 Subject: [PATCH 4/5] Update branch filters to include/exclude 9.* (#42521) This commit updates branch_configuration and branch filters in CI pipeline definitions to support the new 9.* branches. --- catalog-info.yaml | 106 +++++++++++++++++++++++----------------------- 1 file changed, 53 insertions(+), 53 deletions(-) diff --git a/catalog-info.yaml b/catalog-info.yaml index 16d4bd7e0d0b..4ce530093d0b 100644 --- a/catalog-info.yaml +++ b/catalog-info.yaml @@ -39,7 +39,7 @@ spec: name: beats description: "Beats Main pipeline" spec: - branch_configuration: "main 7.17 8.*" + branch_configuration: "main 7.17 8.* 9.*" pipeline_file: ".buildkite/pipeline.yml" provider_settings: build_pull_request_forks: false @@ -51,9 +51,9 @@ spec: build.pull_request.id == null || (build.creator.name == 'elasticmachine' && build.pull_request.id != null) repository: elastic/beats cancel_intermediate_builds: true - cancel_intermediate_builds_branch_filter: "!main !7.* !8.*" + cancel_intermediate_builds_branch_filter: "!main !7.* !8.* !9.*" skip_intermediate_builds: true - skip_intermediate_builds_branch_filter: "!main !7.* !8.*" + skip_intermediate_builds_branch_filter: "!main !7.* !8.* !9.*" env: # TODO set to true once https://github.com/elastic/ingest-dev/issues/3001 has been resolved ELASTIC_PR_COMMENTS_ENABLED: "false" @@ -89,7 +89,7 @@ spec: name: beats-metricbeat description: "Beats Metricbeat pipeline" spec: - branch_configuration: "main 7.17 8.*" + branch_configuration: "main 7.17 8.* 9.*" pipeline_file: ".buildkite/metricbeat/pipeline.yml" maximum_timeout_in_minutes: 120 provider_settings: @@ -102,9 +102,9 @@ spec: build.pull_request.id == null || (build.creator.name == 'elasticmachine' && build.pull_request.id != null) repository: elastic/beats cancel_intermediate_builds: true - cancel_intermediate_builds_branch_filter: "!main !7.* !8.*" + cancel_intermediate_builds_branch_filter: "!main !7.* !8.* !9.*" skip_intermediate_builds: true - skip_intermediate_builds_branch_filter: "!main !7.* !8.*" + skip_intermediate_builds_branch_filter: "!main !7.* !8.* !9.*" env: # TODO set to truue once https://github.com/elastic/ingest-dev/issues/3001 has been resolved ELASTIC_PR_COMMENTS_ENABLED: "false" @@ -136,7 +136,7 @@ spec: name: filebeat description: "Filebeat pipeline" spec: - branch_configuration: "main 7.17 8.*" + branch_configuration: "main 7.17 8.* 9.*" pipeline_file: ".buildkite/filebeat/filebeat-pipeline.yml" maximum_timeout_in_minutes: 120 provider_settings: @@ -149,9 +149,9 @@ spec: build.pull_request.id == null || (build.creator.name == 'elasticmachine' && build.pull_request.id != null) repository: elastic/beats cancel_intermediate_builds: true - cancel_intermediate_builds_branch_filter: "!main !7.* !8.*" + cancel_intermediate_builds_branch_filter: "!main !7.* !8.* !9.*" skip_intermediate_builds: true - skip_intermediate_builds_branch_filter: "!main !7.* !8.*" + skip_intermediate_builds_branch_filter: "!main !7.* !8.* !9.*" env: # TODO set to truue once https://github.com/elastic/ingest-dev/issues/3001 has been resolved ELASTIC_PR_COMMENTS_ENABLED: "false" @@ -196,9 +196,9 @@ spec: build.pull_request.id == null || (build.creator.name == 'elasticmachine' && build.pull_request.id != null) repository: elastic/beats cancel_intermediate_builds: true - cancel_intermediate_builds_branch_filter: "!main !7.* !8.*" + cancel_intermediate_builds_branch_filter: "!main !7.* !8.* !9.*" skip_intermediate_builds: true - skip_intermediate_builds_branch_filter: "!main !7.* !8.*" + skip_intermediate_builds_branch_filter: "!main !7.* !8.* !9.*" env: # TODO set to true once https://github.com/elastic/ingest-dev/issues/3001 has been resolved ELASTIC_PR_COMMENTS_ENABLED: "false" @@ -230,7 +230,7 @@ spec: name: heartbeat description: "Heartbeat pipeline" spec: - branch_configuration: "main 7.17 8.*" + branch_configuration: "main 7.17 8.* 9.*" pipeline_file: ".buildkite/heartbeat/heartbeat-pipeline.yml" maximum_timeout_in_minutes: 120 provider_settings: @@ -243,9 +243,9 @@ spec: build.pull_request.id == null || (build.creator.name == 'elasticmachine' && build.pull_request.id != null) repository: elastic/beats cancel_intermediate_builds: true - cancel_intermediate_builds_branch_filter: "!main !7.* !8.*" + cancel_intermediate_builds_branch_filter: "!main !7.* !8.* !9.*" skip_intermediate_builds: true - skip_intermediate_builds_branch_filter: "!main !7.* !8.*" + skip_intermediate_builds_branch_filter: "!main !7.* !8.* !9.*" env: # TODO set to true once https://github.com/elastic/ingest-dev/issues/3001 has been resolved ELASTIC_PR_COMMENTS_ENABLED: "false" @@ -290,9 +290,9 @@ spec: build.pull_request.id == null || (build.creator.name == 'elasticmachine' && build.pull_request.id != null) repository: elastic/beats cancel_intermediate_builds: true - cancel_intermediate_builds_branch_filter: "!main !7.* !8.*" + cancel_intermediate_builds_branch_filter: "!main !7.* !8.* !9.*" skip_intermediate_builds: true - skip_intermediate_builds_branch_filter: "!main !7.* !8.*" + skip_intermediate_builds_branch_filter: "!main !7.* !8.* !9.*" env: # TODO set to true once https://github.com/elastic/ingest-dev/issues/3001 has been resolved ELASTIC_PR_COMMENTS_ENABLED: "false" @@ -324,7 +324,7 @@ spec: name: beats-libbeat description: "Beats libbeat pipeline" spec: - branch_configuration: "main 7.17 8.*" + branch_configuration: "main 7.17 8.* 9.*" pipeline_file: ".buildkite/libbeat/pipeline.libbeat.yml" maximum_timeout_in_minutes: 120 provider_settings: @@ -337,9 +337,9 @@ spec: build.pull_request.id == null || (build.creator.name == 'elasticmachine' && build.pull_request.id != null) repository: elastic/beats cancel_intermediate_builds: true - cancel_intermediate_builds_branch_filter: "!main !7.17 !8.*" + cancel_intermediate_builds_branch_filter: "!main !7.17 !8.* !9.*" skip_intermediate_builds: true - skip_intermediate_builds_branch_filter: "!main !7.17 !8.*" + skip_intermediate_builds_branch_filter: "!main !7.17 !8.* !9.*" env: # TODO set to true once https://github.com/elastic/ingest-dev/issues/3001 has been resolved ELASTIC_PR_COMMENTS_ENABLED: "false" @@ -371,7 +371,7 @@ spec: name: beats-packetbeat description: "Beats packetbeat pipeline" spec: - branch_configuration: "main 7.17 8.*" + branch_configuration: "main 7.17 8.* 9.*" pipeline_file: ".buildkite/packetbeat/pipeline.packetbeat.yml" maximum_timeout_in_minutes: 120 provider_settings: @@ -384,9 +384,9 @@ spec: build.pull_request.id == null || (build.creator.name == 'elasticmachine' && build.pull_request.id != null) repository: elastic/beats cancel_intermediate_builds: true - cancel_intermediate_builds_branch_filter: "!main !7.17 !8.*" + cancel_intermediate_builds_branch_filter: "!main !7.17 !8.* !9.*" skip_intermediate_builds: true - skip_intermediate_builds_branch_filter: "!main !7.17 !8.*" + skip_intermediate_builds_branch_filter: "!main !7.17 !8.* !9.*" env: # TODO set to true once https://github.com/elastic/ingest-dev/issues/3001 has been resolved ELASTIC_PR_COMMENTS_ENABLED: "false" @@ -430,9 +430,9 @@ spec: build.pull_request.id == null || (build.creator.name == 'elasticmachine' && build.pull_request.id != null) repository: elastic/beats cancel_intermediate_builds: true - cancel_intermediate_builds_branch_filter: "!main !7.17 !8.*" + cancel_intermediate_builds_branch_filter: "!main !7.17 !8.* !9.*" skip_intermediate_builds: true - skip_intermediate_builds_branch_filter: "!main !7.17 !8.*" + skip_intermediate_builds_branch_filter: "!main !7.17 !8.* !9.*" env: # TODO set to true once https://github.com/elastic/ingest-dev/issues/3001 has been resolved ELASTIC_PR_COMMENTS_ENABLED: "false" @@ -464,7 +464,7 @@ spec: name: beats-winlogbeat description: "Beats winlogbeat pipeline" spec: - branch_configuration: "main 7.17 8.*" + branch_configuration: "main 7.17 8.* 9.*" pipeline_file: ".buildkite/winlogbeat/pipeline.winlogbeat.yml" maximum_timeout_in_minutes: 120 provider_settings: @@ -477,9 +477,9 @@ spec: build.pull_request.id == null || (build.creator.name == 'elasticmachine' && build.pull_request.id != null) repository: elastic/beats cancel_intermediate_builds: true - cancel_intermediate_builds_branch_filter: "!main !7.17 !8.*" + cancel_intermediate_builds_branch_filter: "!main !7.17 !8.* !9.*" skip_intermediate_builds: true - skip_intermediate_builds_branch_filter: "!main !7.17 !8.*" + skip_intermediate_builds_branch_filter: "!main !7.17 !8.* !9.*" env: # TODO set to true once https://github.com/elastic/ingest-dev/issues/3001 has been resolved ELASTIC_PR_COMMENTS_ENABLED: "false" @@ -511,7 +511,7 @@ spec: name: beats-xpack-winlogbeat description: "Beats x-pack winlogbeat pipeline" spec: - branch_configuration: "main 7.17 8.*" + branch_configuration: "main 7.17 8.* 9.*" pipeline_file: ".buildkite/x-pack/pipeline.xpack.winlogbeat.yml" maximum_timeout_in_minutes: 120 provider_settings: @@ -524,9 +524,9 @@ spec: build.pull_request.id == null || (build.creator.name == 'elasticmachine' && build.pull_request.id != null) repository: elastic/beats cancel_intermediate_builds: true - cancel_intermediate_builds_branch_filter: "!main !7.17 !8.*" + cancel_intermediate_builds_branch_filter: "!main !7.17 !8.* !9.*" skip_intermediate_builds: true - skip_intermediate_builds_branch_filter: "!main !7.17 !8.*" + skip_intermediate_builds_branch_filter: "!main !7.17 !8.* !9.*" env: # TODO set to true once https://github.com/elastic/ingest-dev/issues/3001 has been resolved ELASTIC_PR_COMMENTS_ENABLED: "false" @@ -558,7 +558,7 @@ spec: name: beats-xpack-packetbeat description: "Beats x-pack packetbeat pipeline" spec: - branch_configuration: "main 7.17 8.*" + branch_configuration: "main 7.17 8.* 9.*" pipeline_file: ".buildkite/x-pack/pipeline.xpack.packetbeat.yml" maximum_timeout_in_minutes: 120 provider_settings: @@ -571,9 +571,9 @@ spec: build.pull_request.id == null || (build.creator.name == 'elasticmachine' && build.pull_request.id != null) repository: elastic/beats cancel_intermediate_builds: true - cancel_intermediate_builds_branch_filter: "!main !7.17 !8.*" + cancel_intermediate_builds_branch_filter: "!main !7.17 !8.* !9.*" skip_intermediate_builds: true - skip_intermediate_builds_branch_filter: "!main !7.17 !8.*" + skip_intermediate_builds_branch_filter: "!main !7.17 !8.* !9.*" env: # TODO set to true once https://github.com/elastic/ingest-dev/issues/3001 has been resolved ELASTIC_PR_COMMENTS_ENABLED: "false" @@ -605,7 +605,7 @@ spec: name: beats-xpack-libbeat description: "Beats x-pack libbeat pipeline" spec: - branch_configuration: "main 7.17 8.*" + branch_configuration: "main 7.17 8.* 9.*" pipeline_file: ".buildkite/x-pack/pipeline.xpack.libbeat.yml" maximum_timeout_in_minutes: 120 provider_settings: @@ -618,9 +618,9 @@ spec: build.pull_request.id == null || (build.creator.name == 'elasticmachine' && build.pull_request.id != null) repository: elastic/beats cancel_intermediate_builds: true - cancel_intermediate_builds_branch_filter: "!main !7.17 !8.*" + cancel_intermediate_builds_branch_filter: "!main !7.17 !8.* !9.*" skip_intermediate_builds: true - skip_intermediate_builds_branch_filter: "!main !7.17 !8.*" + skip_intermediate_builds_branch_filter: "!main !7.17 !8.* !9.*" env: # TODO set to true once https://github.com/elastic/ingest-dev/issues/3001 has been resolved ELASTIC_PR_COMMENTS_ENABLED: "false" @@ -652,7 +652,7 @@ spec: name: beats-xpack-metricbeat description: "Beats x-pack metricbeat pipeline" spec: - branch_configuration: "main 7.17 8.*" + branch_configuration: "main 7.17 8.* 9.*" pipeline_file: ".buildkite/x-pack/pipeline.xpack.metricbeat.yml" maximum_timeout_in_minutes: 120 provider_settings: @@ -665,9 +665,9 @@ spec: build.pull_request.id == null || (build.creator.name == 'elasticmachine' && build.pull_request.id != null) repository: elastic/beats cancel_intermediate_builds: true - cancel_intermediate_builds_branch_filter: "!main !7.17 !8.*" + cancel_intermediate_builds_branch_filter: "!main !7.17 !8.* !9.*" skip_intermediate_builds: true - skip_intermediate_builds_branch_filter: "!main !7.17 !8.*" + skip_intermediate_builds_branch_filter: "!main !7.17 !8.* !9.*" env: # TODO set to true once https://github.com/elastic/ingest-dev/issues/3001 has been resolved ELASTIC_PR_COMMENTS_ENABLED: "false" @@ -782,9 +782,9 @@ spec: build.pull_request.id == null || (build.creator.name == 'elasticmachine' && build.pull_request.id != null) repository: elastic/beats cancel_intermediate_builds: true - cancel_intermediate_builds_branch_filter: "!main !7.17 !8.*" + cancel_intermediate_builds_branch_filter: "!main !7.17 !8.* !9.*" skip_intermediate_builds: true - skip_intermediate_builds_branch_filter: "!main !7.17 !8.*" + skip_intermediate_builds_branch_filter: "!main !7.17 !8.* !9.*" env: # TODO set to true once https://github.com/elastic/ingest-dev/issues/3001 has been resolved ELASTIC_PR_COMMENTS_ENABLED: "false" @@ -829,9 +829,9 @@ spec: build.pull_request.id == null || (build.creator.name == 'elasticmachine' && build.pull_request.id != null) repository: elastic/beats cancel_intermediate_builds: true - cancel_intermediate_builds_branch_filter: "!main !7.17 !8.*" + cancel_intermediate_builds_branch_filter: "!main !7.17 !8.* !9.*" skip_intermediate_builds: true - skip_intermediate_builds_branch_filter: "!main !7.17 !8.*" + skip_intermediate_builds_branch_filter: "!main !7.17 !8.* !9.*" env: # TODO set to true once https://github.com/elastic/ingest-dev/issues/3001 has been resolved ELASTIC_PR_COMMENTS_ENABLED: "false" @@ -876,9 +876,9 @@ spec: build.pull_request.id == null || (build.creator.name == 'elasticmachine' && build.pull_request.id != null) repository: elastic/beats cancel_intermediate_builds: true - cancel_intermediate_builds_branch_filter: "!main !7.17 !8.*" + cancel_intermediate_builds_branch_filter: "!main !7.17 !8.* !9.*" skip_intermediate_builds: true - skip_intermediate_builds_branch_filter: "!main !7.17 !8.*" + skip_intermediate_builds_branch_filter: "!main !7.17 !8.* !9.*" env: # TODO set to true once https://github.com/elastic/ingest-dev/issues/3001 has been resolved ELASTIC_PR_COMMENTS_ENABLED: "false" @@ -923,9 +923,9 @@ spec: build.pull_request.id == null || (build.creator.name == 'elasticmachine' && build.pull_request.id != null) repository: elastic/beats cancel_intermediate_builds: true - cancel_intermediate_builds_branch_filter: "!main !7.17 !8.*" + cancel_intermediate_builds_branch_filter: "!main !7.17 !8.* !9.*" skip_intermediate_builds: true - skip_intermediate_builds_branch_filter: "!main !7.17 !8.*" + skip_intermediate_builds_branch_filter: "!main !7.17 !8.* !9.*" env: # TODO set to true once https://github.com/elastic/ingest-dev/issues/3001 has been resolved ELASTIC_PR_COMMENTS_ENABLED: "false" @@ -970,9 +970,9 @@ spec: build.pull_request.id == null || (build.creator.name == 'elasticmachine' && build.pull_request.id != null) repository: elastic/beats cancel_intermediate_builds: true - cancel_intermediate_builds_branch_filter: "!main !7.17 !8.*" + cancel_intermediate_builds_branch_filter: "!main !7.17 !8.* !9.*" skip_intermediate_builds: true - skip_intermediate_builds_branch_filter: "!main !7.17 !8.*" + skip_intermediate_builds_branch_filter: "!main !7.17 !8.* !9.*" env: # TODO set to true once https://github.com/elastic/ingest-dev/issues/3001 has been resolved ELASTIC_PR_COMMENTS_ENABLED: "false" @@ -1017,9 +1017,9 @@ spec: build.pull_request.id == null || (build.creator.name == 'elasticmachine' && build.pull_request.id != null) repository: elastic/beats cancel_intermediate_builds: true - cancel_intermediate_builds_branch_filter: "!main !8.*" + cancel_intermediate_builds_branch_filter: "!main !8.* !9.*" skip_intermediate_builds: true - skip_intermediate_builds_branch_filter: "!main !8.*" + skip_intermediate_builds_branch_filter: "!main !8.* !9.*" env: # TODO set to true once https://github.com/elastic/ingest-dev/issues/3001 has been resolved ELASTIC_PR_COMMENTS_ENABLED: "false" @@ -1053,7 +1053,7 @@ spec: spec: repository: elastic/beats pipeline_file: ".buildkite/packaging.pipeline.yml" - branch_configuration: "main 8.* 7.17" + branch_configuration: "main 8.* 7.17 9.*" cancel_intermediate_builds: false skip_intermediate_builds: false maximum_timeout_in_minutes: 90 @@ -1102,7 +1102,7 @@ spec: spec: repository: elastic/beats pipeline_file: ".buildkite/ironbank-validation.yml" - branch_configuration: "main 8.* 7.17" + branch_configuration: "main 8.* 7.17 9.*" cancel_intermediate_builds: false skip_intermediate_builds: false provider_settings: From 0ff52ebd4dd6fee948f790091248ae9cfbef0cb8 Mon Sep 17 00:00:00 2001 From: William Easton Date: Fri, 31 Jan 2025 01:59:25 -0600 Subject: [PATCH 5/5] Write latency statistics go under a histogram key (#42514) * Write latency goes under histogram * Update fields.yml to include new exposed beats stats * Add't updates for integ tests --------- Co-authored-by: Julien Lind --- metricbeat/docs/fields.asciidoc | 134 ++++++++++++++++++ metricbeat/module/beat/fields.go | 2 +- metricbeat/module/beat/stats/_meta/fields.yml | 78 +++++++++- metricbeat/module/beat/stats/data.go | 11 +- 4 files changed, 216 insertions(+), 9 deletions(-) diff --git a/metricbeat/docs/fields.asciidoc b/metricbeat/docs/fields.asciidoc index 58ee16f2c838..3d571d94d63a 100644 --- a/metricbeat/docs/fields.asciidoc +++ b/metricbeat/docs/fields.asciidoc @@ -10184,16 +10184,113 @@ type: long -- + *`beat.stats.libbeat.pipeline.queue.acked`*:: + -- +Number of acknowledged events + + +type: long + +-- + +*`beat.stats.libbeat.pipeline.queue.added.bytes`*:: ++ +-- +Number of bytes added to the queue + + +type: long + +-- + +*`beat.stats.libbeat.pipeline.queue.added.events`*:: ++ +-- +Number of events added to the queue + + +type: long + +-- + +*`beat.stats.libbeat.pipeline.queue.consumed.bytes`*:: ++ +-- +Number of bytes consumed from the queue + + type: long -- +*`beat.stats.libbeat.pipeline.queue.consumed.events`*:: ++ +-- +Number of events consumed from the queue + + +type: long + +-- + +*`beat.stats.libbeat.pipeline.queue.filled.bytes`*:: ++ +-- +Number of bytes filled in the queue + + +type: long + +-- + +*`beat.stats.libbeat.pipeline.queue.filled.events`*:: ++ +-- +Number of events filled in the queue + + +type: long + +-- + +*`beat.stats.libbeat.pipeline.queue.filled.pct`*:: ++ +-- +Percentage of the queue filled + + +type: float + +-- + *`beat.stats.libbeat.pipeline.queue.max_events`*:: + -- +Maximum number of events allowed in the queue + + +type: long + +-- + +*`beat.stats.libbeat.pipeline.queue.removed.bytes`*:: ++ +-- +Number of bytes removed from the queue + + +type: long + +-- + +*`beat.stats.libbeat.pipeline.queue.removed.events`*:: ++ +-- +Number of events removed from the queue + + type: long -- @@ -10437,6 +10534,43 @@ type: long +*`beat.stats.libbeat.output.write.latency.histogram.count`*:: ++ +-- +type: long + +-- + +*`beat.stats.libbeat.output.write.latency.histogram.max`*:: ++ +-- +type: float + +-- + +*`beat.stats.libbeat.output.write.latency.histogram.median`*:: ++ +-- +type: long + +-- + +*`beat.stats.libbeat.output.write.latency.histogram.p95`*:: ++ +-- +type: float + +-- + +*`beat.stats.libbeat.output.write.latency.histogram.p99`*:: ++ +-- +type: float + +-- + + + *`beat.stats.output.elasticsearch.bulk_requests.available`*:: + diff --git a/metricbeat/module/beat/fields.go b/metricbeat/module/beat/fields.go index f0a40fd701af..45e1586c4e26 100644 --- a/metricbeat/module/beat/fields.go +++ b/metricbeat/module/beat/fields.go @@ -32,5 +32,5 @@ func init() { // AssetBeat returns asset data. // This is the base64 encoded zlib format compressed contents of module/beat. func AssetBeat() string { - return "eJzsXUtv5LgRvs+vIHzOCpjDXHzYBIskwF4SIFgghyAw2Orqbq4pUkNSnnV+fSCpW60HHyWJktUz8mUX9uirr0pSPSiy6ifyCu/P5ADUfCLEMMPhmTz9AtQ8fSLkCDpVLDdMimfy8ydCCCn/RDJ5LDh8IkQBB6rhmZzpJ0I0GMPEWT+T/zxpzZ/+RJ4uxuRP/y3/dpHKvKRSnNj5mZwo1+X1Jwb8qJ8r5J+IoBnUXPSLNtTo6veEmPe8FKFkkV9/076ufS3Ns580qDdQzZ9sl9sgOjBp1vm9C8OF08ZS8LUAbZJUFsIM/tUNmXJGteWvOTWX2iRJZZGE5tlLrWFC0yzxo9856FyKyuJ28TbFfMq1wV2KhZXDKVhTd9rvTgSUksolx69nSNeOvlxqODr/GUbpUYrXaiUBsZi7sSA5r9QbtyOk8mh7CJclFxB7Y3eS6sCORxCrEwxLvnFkwoASlK9OMSj4zvCNcnb8WoB6/wCWCOE3phmYi1z/PQ6IvbET0pxkIdbnFxR8Y/i1gGL9l9kvtQl41ABnGVvfE4Yl3zgaloEs1mcYktvwk5JTdV7/FgcF3xgWghbmIhX73wfEY5TwFtM3yjg98PXNiZF941m5T2rWJ+kV3GG3eH5H0xRys+YTVemVBOV+UIpXs8NleEKaTB7Zia1vPozoG035ujY7i8S7b9AQvzAcojbP9xmESU/naBUucKoNSzVQlV6WKTFpeoEEhFEM/MXGzILzapqko1KCFd+lq+CkQF+SE2W8UOAvTBdhjGJgJ62LNAWtP5K1n0JTtYFJL8m6PL0yu8ROlPMDTV3+ZkF6XsldkteyaX2OPsFdiuEkZkGaLuE3ir9TOHdWHMksf3pWebrQSh3nkPpdZ4wMKrTk2ZU4J9LWlk9KiyVX7QJLokOuzqXRLk2fYQjCOGREBkfQxiGTDBRcUu0T9i6tdhmH7ESQtiIj7UVG2YzMsRtq0bOvA2ZJ4mP0wDPDL118jCZYXiMXED5GGTS5TnmMeipX8C0hMjfSmmY5Z6JfjXSZxghK8Aai5JcCe4OwmeLGppuWySgWHxtPG857QA1baGJE/aC3dUgcTWcLzmZI38WmWfAxvP+obTA/5/K8/Ofr9ZxJafT6hpWK7W7Ea5s9JScTTfZA2XhAhcdIxANKPEwOHtBj8+m3iz82887AKJausF9KCl1koJJC6CLPpTJwfDkqmecLfyS6G+iqajKJykcGzhvxPXYGzLOHTzLdag8ZQV1aPFoQdenxgHHUpcoDhdKBCthoahRNA18hH7V6q1XbY1DAOnsIIpON9pARyKHEowUghxoPGH8cmjxQ+Olr4GNyY3sxxuYa9uVIm5VLY+3LkWHb7MGMTDTZo4UytwoPFMjcSjxWGHPr8RhBzMJ/X44cGOgRlyM7xPfYGTDPHj7JdKs9ZAT9DpYjvXo8YBx97OVIuwo/9nJkZZN9ORJlnT0EkclGe8gI9PjLkT41HjD+PPRypFWD0DbLXMkUtJbxzkJVNit5CH2SKqOGSWHTbfph2YZzgpPVrVdXooYU1myqz6lYiRlKVMPLKKD91lokymJ14PT8vNOBLXUrDfxn9VfuR+U/vYlTfrQBug13ghxX6SXiIukU3kqWhaZp+eyu9NZgJTa5hZR93z/dpy7Xka6k+X23pLtq+AP2pOtqvrWmdHZ2W+lKZ2W3qbZ0VoYb60tn5bi9xnQOmtvqTGcluaXWdFaCG+pNZ+W3peZ0VoLb6k5npbip9nR2hpvrT2eluckGdS6m2+pQZ2X5o7Wo6xphcz3qbPS21aTOxnBDXeps9FZtU1cRcPepa055t4pl7dge4SbhI4CX0HDq95YnG63Ebyb+nmvxRscfsBrv6761etzFbysVuYPfpmpyB8eNVeUOltury51Et1WZO2huqTZ3UNxQde5guKX63EFxWxW6g+SmanQXx81V6Q6im6zT3Vy3Vak7eP5otXrfDJur1u0Et1Wv2zluqGK3E1y1Zr/+pwt8E3eop8h1pYyZu3aR2v6ZG1XZM3GSyQCieYpo1ncHY7EHEE2ce8/nYg8gmntYDJzUWOwBROMBQWkm+4XEWPg+ivtEz/gnQr9rA/GG8aV5kaTS3gp/5EtRM0vciPdjtPSYfI4m0AHXkfYlrrQhXFe3yOIseB15Qqpl9pA9fX6Kt1DV1qhkbH0E2rK/LCrcdo9a0hcV/sX6anN26AUMMuvdzlkOnImFVjE5A2H8S4ejjHTVPrmxTnwSOoVzQtPXmPscB0xCUprV1DcIrOzGyZ8Ne1ugiBioXamjk4C8Zo1yqROjLl7Yc6EnyviavALyGlqMG1CrEgtJbBxHceBMX9bkFhZ5/zZjllgXdRHzi7uv65glVpVdpOzimqKrMLl1LWq+16/lr+Di3A6dzDdnbaC7f/MJW8vr9jmhfO6BmvTiPNUYnVVI3GqhoEsLGwiORc5ZShdZenIwu0kM22zpKNUlhoxRRsqMigW9bZdVSNxK3rZPyiXsHpHoMTm8u+/yjBz0yiUgosME8U1/NhWPDM8SUDwSTvCb9G+KGVj4poRkdLksfFu8QlqtXk7MNllmfmaQyWPBIVGFEKHhNZPUrKl78XtUtKFqiQL1ysQDPyAi8yV59NEdq89k1mJCtbAJ+QUyUJS/WL8TjVziCkN2hBe5YRkkWYSlygBes/pOxZF7lkZn9RJkGTPJhaqICxdXvkkAu0tBy1PE3WhdCk7splTJnbt2Zgi3ot4X4bPqXy9yV8/pi4A/ItrzxjbxIbdUk+r9hXIubfN45lKo4RM3fI+HL02LwAORmOmIbrcRb0NtAqzr4Ymw4JoXi9fdSxyark1SfRZCnItOTzrJQTF5TIoF6toWGZykNq+vhTR0DVpBQff2AXZH1qUTs4NPbTGfTII2AwmYov4FRmJTc1yUNIbDEXVVXJ7jZA8ZW4/WL0/WIbbldWiaGverO8GT3tn4oPsMKnfvttJ8Hl4B3dC2hDn8yC3519wqcm3bZYER06ZUaHqOXm4PKfnE+OPk/Ph7/YhrWPoaU8fylQwhDyi4yqAYJNzYneW35I1y59bhiRxCwF0CC9yJEHCfwAL3IQzd7IPSoJYwQgC3J34BE3SQP/XlDr5ujd9DBZxqw1INVKWXaCskh4K/vlzPly1TXYa2Gk+w922tu22RpKNJ4pd6X1rMcg5R+00huLml3peOjvAHWBdeI9wQ32e6WBrfNPB9FGzuggLP963ojHzi7mfXtFHyfT1SXYHtxVD9Ul5/s+Dw9vdv+Sa28UJS/e8i23iv2Its471iL7KN94q91DbeK7xrG28ZmrShWT54JvrYNe7TX5orngZg7SeSTHq47Lcw4lPgfHqnRMnyXWUGUlMo98ZE/0J+ySfx4rSZWywzSpATw/MCjhLgBZfLBK6cU3Oy7xoOcye2R6hSRerEi+x/PyNItwHfM4EyaNhO2Ix4U1wgXSHl//ehRgsZgvS+K85UxYnSEzNTGSdKN6Gf6cFcII2bBfXGUruASZ9DZ3//hOTKybbkE8G3WAR5PY37nZworQ1oyaSGtj+CThXLDZPimfwcyMk6RquxXuH9m+x887Qi1j+/ADXk178mw+jeTVimQpfXDcG7KWvKC21AJQhdxqULIXJ9FAUcqC6BKDLpWPF9Gtq+e6n3HQldHHrqbdd3fH00C/hb3nBp2fTStoA7T/CpEHWLiCVP9LPo6zD5Ysd2MNzFlkIEf/Gc5+f2+jsrId/FUZPyVApDmQDXkXucLRbKVV9BCfDvI3AxI+EHbBwIKm/GAGEyYN/tz6igZ8hAmAQEPQw389YQByk50L4YT3gof37VJAVhFOUtMeQq5s/TMlImDJwHjcICRP5RZAdQRJ6u+JrYdEWnqz57hpJQ+z0N8acZlOxrcFJoOJLDexWCrSTqc2bLcKiwyQGYONuJuPYUTM8w9KwM437GPpqbo+kyx2Qxk5c8YbwL5R3LFG+rD67DhZNvGyo4Jyn+NpRgQzmkCmSERSZABvu2TcDEtFqbAItojTYJFdXLbAJysPHYBExEr7AJqKHmXhMgsSOsRsLiRkqNBQ23vJqAOmKa1XhkREOpCcCI9k9I1A7iqn4X0bsJqQNZ1vPiWiRNAPZ2NELg3R8zeyeiAERzI84gTHpa5iyPb38ETkpIUltaStMLJGXBwSA8YnDE61EDKzgp0JfqyGNhb3QzG1sXaQpaxwI/gUkvSVy0E+X8QNNwP64RmNhBWyMgx7Tz874gv1NLvUmivB5nlfuPV8TJzzmHNPwmxPTu2OG1ZELGEh43i9eJLD5NNagXGVP/dLEx2pERGpIJWpJxmpIJuedMMfiZqzOEjJyJOlJSJ01Dzs4d5TJv7fpXdRLVsfhEQQrsDcZMYR3hK3Y3NNkNxXzO+hKiPchNMm34MoflVgnRXJ7XXfXaX4w9PgdF7PF5M/F52O/VDhZ1ZVwKXWR1R+Iiz6UycHTMCJqgGdm9kYXB7o2cInZvtBlvZBRNEWtKe8Li/9ldhJ/N7iIey0U0G8qMcT1Ee8lkh9490O6BRojZPdBeMu3eaPdGARG7N9qMN9pLJu/P7iJsVHYX8T27iOYkiJIpaC2X2WJSPc71FPaTVBmtBrF7q7NgfVdnVnExdU5FZESjgPqP16w6g3KrG+DDW6+QOpDoW3VbwVNompYPxfxnpPF8Ui4z2WQ/w7Gf4bBC7mc49jMc+xmO/QzHPY3cz3BYUPczHGiI/vbNpJUpac/CIwbSdmCW7AmS42dPkCJA7gnSniDtCdKeIO0Jkhd1T5BQEHenepKfbFdMafwh5jSXmtXWyjxkZ6qobcF24+Oub3L4ahJDNPNXQ4Wk/eAzoqLgkrrNMKeeePr85PUMR1mEes4/ff4yH2M+hAg1BItRyrjNheJKcGYbjzUNqvVwRnvQA2NUMNVzaAYKAsM/wwQNMEOL0AQRBIR3BAj++pEMFgn9qJGlwVgQHD3qVMY6EXCWN1/EG2Om4SGTPsQEOyRSYMoWps+ib0JePOccnvg2ZtPRiEFyk2C9k+YQb7d3cBvuhULMXEMQqSeDLfI+RHj0xoxqQ+zewA9Zc/oix+jb6d7IPZQWffec42TxCK5BsJjtBNYJsQEz2sdCT7eiYwgyJjn3DXpGX++Y0hywgismOsUGGpVWbUNrUKs8VYhK4FkqWRgmHDdgvOB7l9k7NLnOmSdMuNu1Xgexo5+DAI+/V48JSWWWSUGMJJTzSnhf0dDzlLMcOBPu/HNWosAZiEBIDXqzutEtTV8DU6GQQBn946VqSzKTFgIjzuYzz9QwBN02VviYwgiwE2XDRtJTsRg3oGKh5cWBM32JBafAeD/VjEmrPEPOA1D3lWFxYjZJET7a1m5s3muhDVVzXy1tZD4TQgGX1JkNI+w8mBzZvdZl54DbLn/+WTf4dhUbmDvlWKskqJQTRbL8+e09b3UkX9IZIgn9rZRSfxuxD2kkoxyrO6AQ7FcQJHHSyRxqY5UEhPzG4Xj28IgZBibSrUX7b3HrawA16WVuN8o5hg0yiBoN5z0CIQoN1SLnLKXuWm4dtjcWYcIxkoN5ZAMMWt/iMypmhveJTI2UpBSOfbVmZA+TmP5WyiOiZ9lA2LV+ACIxo8C/gB69S3XYCOBbHCGL3vlKss9Yo7eSLcGy5OeTf6P4TTHnHo5od/3fpZDv4raX5jKezWobuPPVHbUR8GbHM767BHqIz69tQvPWw1JCktrSZral7sOFxqUj4O6fy5wTzslm1y38g8pHgoVGjPfg/h8AAP//94XAGA==" + return "eJzsXUuPpLiW3tevsHI9jVSLWnQuekatmZF6MQ+NWprF1VXKASci3GkwZZusyvvrr4CA4OHHAQxJVJGbblUm3/nOAc4L+/gX8grvz+QEVH8iRDPN4Zk8/Q5UP30iJAEVS5ZrJrJn8tsnQggpf0VSkRQcPhEigQNV8Ewu9BMhCrRm2UU9k789KcWf/oU8XbXOn/5e/u4qpH6JRXZml2dyplyV158Z8EQ9V8i/kIymUHNRL0pTrap/J0S/56UIKYr89i/d67rX0jz9RYF8A9n+ynS5CaIHE6e9f7dh2HC6WBK+FqB0FIsi06O/apApZ1QZfptTfa1NElUWiWievtQaRjROIzf6nYPKRVZZ3CzepJhLuS64TTG/cjgFa+pW+92JgJRC2uS49fTp2tOXCwWJ9c8wSk9SvFYr8ojF3I0VyTmlNtwSiEViegjXJecR27A7C3liSQLZ5gT9khuOLNMgM8o3p+gVfGf4RjlLvhYg3z+AJUJ4wzQFfRXbv8cesQ27TOizKLLt+XkFNwy/FlBs/zK7pbYBj2rgLGXbe0K/5IajZimIYnuGPrktPyE4lZftb7FXcMOwyGihr0Kyf3xAPEYJ7zB9o4zTE9/enBjZDc/KfVK9PUmn4B671fM7GseQ6y2fqEqvyCv3g1K8mh0uw8uETkXCzmx782FENzTF69bsDBLvvkFB+MJwjNo+3xfIdHy+BKtwgVOlWayAyvi6TolJ4ytEkGnJwF1sLCw4b6aJeipFWPF9uhLOEtQ1OlPGCwnuwnQVxigGZtKqiGNQ6iNZuym0VRvo+Bpty9Mps0/sTDk/0djmb1ak55TcJ3krm7bn6BLcp+hPYlakaRPeUPyLwqXXcSSL/OlF5vFKnTrOIXa7zhAZlK/l2Ze4JNLWlo9Ki0U37Twt0TFXa2u0T9NlGIIwDpmQwRG0ccgsA3lbqkPCztZqn7HPTgRpKzLRXmSSzcgSu6GankMdMC2Jj9EDzwzfuvgYTbC8JjYQPkYZNLleeYx6KjfwLT4yDWlF05yzbFiN9JmGCErwBlnJLwb2Bn4zhY1NjZbRJBYfG09bzkdA9VtoZkT9oLd1TBxNZw/OZkzfxqZt+Gg+fNR2mJ9zcVn/8/V2zqQ0en3DSsUON+K0zZGSk5kme6Bs3KPCYyTiHiUeJgf36LH79NvGH5t5p6AlizdYLyUyVaQgoyJTRZ4LqSF5SaTI85U/Et0NdFM1mkXlIwNnQ/yInR7zHOGTzLfaQ0ZQmxaPFkRtejxgHLWp8kChdKQCNppqSWPPV8hHrd5q1Y4Y5LHOEYLIbKM9ZASyKPFoAciixgPGH4smDxR+hhq4mDRsr1qbXMPRjjRZuTTW0Y702+YIZmSmyR4tlNlVeKBAZlfiscKYXY/HCGIG/kc7cmSgR2xH9ogfsdNjniN8kvlWe8gI+gO0I516PGAcfex2pFmFn7sdWdnkaEeirHOEIDLbaA8ZgR6/HelS4wHjz0O3I40a+JZZ5lLEoJQItxeqslnJI1NnIVOqmchMus3fLNtyjnCy+vXqRtSQwtpF9TnNNmKGEtXy0hLocLQWCdKs9uyeX7Y7sKNupYF7r/7G86jcuzdxyk82QH/gjpfjJrNEbCStwjvJcqZoXD67G701WIltbiHE0PfP96nrTaQraf7YI+luGv6EM+n6mu9tKJ2Z3V6m0hnZ7WosnZHhzubSGTnubzCdhea+JtMZSe5pNJ2R4I5m0xn57Wk4nZHgvqbTGSnuajydmeHu5tMZae5yQJ2N6b4m1BlZ/mwj6vpG2N2MOhO9fQ2pMzHc0ZQ6E71Nx9RVBOxz6tpd3p1iWVmWR9hJuAjgJbSchrPlyU4r8cbEP3It3ur4E1bjQ933Vo/b+O2lIrfw21VNbuG4s6rcwnJ/dbmV6L4qcwvNPdXmFoo7qs4tDPdUn1so7qtCt5DcVY1u47i7Kt1CdJd1up3rvip1C8+frVYfmmF31bqZ4L7qdTPHHVXsZoKb1uy3//SBG3Gn+hS5vpQp565dhTJ/5kZV9iw7i2gE0T5FNB26g6nYI4g2zr3nS7FHEO09LEZOair2CKL1gCAVE8NCYir8EMW+o2f6E6HelYZwh/HFeRHFwjwKf+JLUTOL7Ij3bbQ0iT4HE2iB60n7ElbaGK6vW2BxBryevEzIddaQPX1+Cteo6mpUMjY+Al3ZX1YVbrpHHemrCv9ifLU5Ow0CBln0bucsB86ylbqYnEGm3a3DSUa6aR81rCOXhF7hHNH4NeQ6xxETn5S2m/oGns5umPxZs7cVioiR2pU6KvLIa3uUa+0YtfHC7gs9U8a35OWR19JiXIPclJhPYus4ihNn6rolN7/I+7cZvUZf1EbMLe7e19FrdJVtpMzi2qKr0LmxF7Xc69fyN3BxdodOlpuzNtDdv7mEbeV1h5xQPvdEdXy17moMzsonbrNQ0KeFDQRJkXMW01VaTxZmjUS/zdaOUn1iyBilhUhptqK37bPyidvI2w5J2YTdIxJNotO7/S4vyEFvXDwiekwQ3/QXU3HIcLSAwpGwgjfSv0mmYeWb4pPR57LybXEK6Yx6OTPTyTLLM4NUJAWHSBZZ5ju8ZpaaNXUn/oCK0lSuUaDemDjgR0REviaPIbql+0wWNROqxibkV0hBUv5i/E40scXlh+wJL3LNUojSAK1KD17bfadZwh2t0UWzBFnKdHSlMmDj4sY38mD3KShxDrgarU/Bit2WKrl11c4C4UbUexM+rf56lbt6iV8y+B7Qng3byIXcUU3I9xfKuTCdx7OUQg0f2eEHPFxpWgAeiMRMBXS7rXgTahtgbQ9PgIZrXqxed6+xabo2SfVZCLEvOj6rKAfJRBIVK9S1HTI4SV1eXwuh6Ra0vILu4wPMjqxPJ+QEn9piLpkEbQbiMUX9DxiJbc1xlUJrDgnqqrA8p8keMzZurV+frEVsx+vQONb2V3eGJ72zcUEPGVTu3m6l5TycAvqhbQ1zuJE78m+5VeDats8CI6ZLqVD0ErzcHlNyiXHHyeXx9/YRV7P4NaSO5SvpQx5RsJVBIUjYsXvtt+iNcuvS4ZkcfMB9AivcCR/wkMAK98EP3a6DUiDXMIIHdyB+BRP0kD8N5Y6+bk1fQwWcKs1iBVTG12AdklPBX19u+8vWqS59S41n2LvpdXctEvU0idxS763FNOcQdN4Ugptd6r11lMB3MDZeA9wQ12e6UBo3Grg+CrZ3QYLj+1ZwRi5x971rSkvxvh2pvsBuM1S9lNc3Fhzf/uEt38UyXoiq/11lGe8Ne5VlvDfsVZbx3rDXWsZ7g7ct4y1Dk9I0zUfPxBC7xn36t/aKpxFY94kksx4u8y0M+BRYn945UbJ8V5mGWBfSvjDR3cgv+UROnC5zg2UmCbJiOF7ASQKc4GKdwJVzqs/mVcN+7sT0CFWqCBU5kd3vZwDpJuB7JlAGDdMOmwlvig2kL6T8/yHUZCFjkMF3xYWqWFEGYhYqY0XpJ/QLPZgNpHWzIN9YbBYw63Po4u+fEN04mVo+AXyLQZDT09jfyZnSuoCGTGps+wRULFmumcieyW+enKxntBrrFd6/id43TyNi/fM7UE3++PdoHN37Cctc6PK6MXg/ZY15oTTICKHLtHTBR26IIoEDVSUQRSYdG75PY9v3L3W+I76LfU+96fqerw9mAffIGy4Mi166FrDnCS4Vgi4RMeSJbhZDHWZfbFkOhrvYUIjgL17y/DSvv7UScl0cNCmPRaYpy8C25R5ni5Vy1VeQGbjXEdiYEf8DNg0ElTdjgDAZsOv2pzSjF0gh0xFk9DRezFtDnITgQIdiHOGh/PlDkRgyLSnviCE3Mf86LyNlmYbLaFCYh8h/F+kJJBHnG74iJl3R6arLnr4k1HxPffxpCiX7GpwUChJyeq9CsJFEvc9sHQ4VNjkByy5mIrY1BfMzDLUow7jvsQ/m5mi8zjZZzMlLjjDeh3IeyxRuqQ9uwoWVbxfKe05S+GUo3oFySBXIBIvMgPTObZuBiRm1NgMWMRptFipqltkMZO/gsRmYiFlhM1B9w71mQGKPsJoIiztSaiqof+TVDNQJp1lNR0YMlJoBjBj/hETtIW7qdxGzm5A6kHU9L25E0gxg50QjBN79MTNPIvJAtDfiApmOz+vs5XGtj8BJ8UnqSotpfIWoLDgY+I8YnPB61MASzhLUtdryWJgH3SzGVkUcg1KhwM+g42sUFu1MOT/R2D+PawIm9qCtCZBTxvk5X5C/qKHeJEFej4vM3dsrwuTnnEPsfxNCenfs4bVkRsbiP24WrxNZ/TRVr15kSv3Tx8ZoRyZoSGZoSaZpSmbkngvF4M9cXSBk4pmoEyX10jTk2bmTXGYzrn9TJ1Fti48kxMDeYMoprBN8xeGGZruhkM/ZUEKwB7lNpjVfZ7PcJiGai8u2Xa/jxTjis1fEEZ93E5/H817NYEE74yJTRVpPJC7yXEgNieWMoBmakcMbGRgc3sgq4vBGu/FGWtIY0VM6Ehb3z+Ei3GwOF/FYLqJdUKa17SE6SiYz9OGBDg80QczhgY6S6fBGhzfyiDi80W680VEyOX8OF2GicriIH9lFtDtBpIhBKbHOEpPqca5PYT8LmdLqIHZndeat7+rMKiymymkWGFFLoO7tNZueQbnXBfD+pVdIHUjwpbqd4JkpGpcPxfJnpPV8Qqxzssmxh+PYw2GEPPZwHHs4jj0cxx6Oexp57OEwoB57ONAQw+WbUSdTUo7GIwbStGGWHAmS5edIkAJAHgnSkSAdCdKRIB0JkhP1SJBQEHenehafTFfMGfyRLRkutWislX7IyVRBx4Idxsdd3+bw1UkMwcxfHSokzBufERUFF9RuhiX1xNPnJ6dnSEThmzn/9PnLcozlEJlvIFiIUsZuLhRXgjPbdKx5UJ2HM9iD7jlGBVM9+85AQWC4zzBBAyzQwneCCALCeQQI/vqJDFYJ/agjS72xwHv0qFUZ44mAi7z5Kt4YcxoeMulDnGCHRPKcsoWZs+g6IS+cc/af+DZl0dGEg+RmwTpPmkO83c6D23AvFOLMNQSR+mSwVd6HAI/elKPaEKs38IesWX2R5ejb+d7Ifigt+u5Zj5PFI9gOgsUsJzCeEOsxo/lY6PlWtByCjEnOXQc9o6+3nNLssYItJlrFegaVVmNDa1CjPFlklcCLkKLQLLPcgOmC71Nm79Dkds48YZl9XOvtIHb0c+Dh8Z/VY0JikaYiI1oQynklfKio73nKWQ6cZfb8c1GiwBlknpDq9WaubnHIRV6vS1d4kduyuZf6b6senvH7WvPjucfdn/tzR+PXTHzjkFwgqeer+I/+pUkCiTMY7ErFimhNuny09RU8nwz6enqMsiNFa6ZzNL1tQ3m4m9rwJmcp0jn6PtzNnavxmXG+yv290ML57WnB3a05l3Fwsp4r3NdVFL3d1QWa5ohBfWcuRrnCFnr+L8gYMk0v1Rj4VrUbda+GKf3+su8b+V/0O0uLlGQjH8y5+Db1jkpIxdvjOeEb7akeqdH24VzwBH3bTqBLx5BZp+NMWDKtZ+PfhDoB7EyZ64WfhsW4BhkKLS9OnKlrKDgJ2rkQZ0rTzNJaQEB10pwzM0kKsCSvLlKX1WJKU7m0nlNa5AshJHBBrb1OhJ1H54L3r7XZGeGE/qc+vsXWSsbcKcuXaIJqKKI95Z/veee8mTWdIZLQf5RSmkhgk7ZpOb+opuuU6ZuEgdnthFI0tpFwojq+Lk915hvWyyBoNFz2CPgotFSLnLOYBkghF7FtWPgJh0gOFpZfbgadlZYpzRaG95lMtRCkFI59tRZkD7OY/lnKG9VBnrBrXN5DQkaB/wOaOD/EYiNAkJpsUZFlNdbkjQJrsCz5ueQ3FL9JZl2hG+yu/38p5Ie47aW5tGMrwg7ufHVHfQTa71ZUQxb7XWjINdVXprS4SOvW/mmSsdLJxPXXBL9UmvR7ZGhgXxtwgAwJGx3fGYh1/uuXlVjnv/66ANlZ0C1YCOY51Gh5OX4q+OvLbZvcBn2mZefkDOFikeYcFo3EuK/fS+C7vd7bZ6stluBMlKf17UBpKd6xcP8MAAD//zT3f3k=" } diff --git a/metricbeat/module/beat/stats/_meta/fields.yml b/metricbeat/module/beat/stats/_meta/fields.yml index 8c1d8973e815..bb48a6c7b651 100644 --- a/metricbeat/module/beat/stats/_meta/fields.yml +++ b/metricbeat/module/beat/stats/_meta/fields.yml @@ -561,10 +561,64 @@ fields: - name: clients type: long - - name: queue.acked - type: long - - name: queue.max_events - type: long + - name: queue + type: group + fields: + - name: acked + type: long + metric_type: counter + description: > + Number of acknowledged events + - name: added.bytes + type: long + metric_type: counter + description: > + Number of bytes added to the queue + - name: added.events + type: long + metric_type: counter + description: > + Number of events added to the queue + - name: consumed.bytes + type: long + metric_type: counter + description: > + Number of bytes consumed from the queue + - name: consumed.events + type: long + metric_type: counter + description: > + Number of events consumed from the queue + - name: filled.bytes + type: long + metric_type: gauge + description: > + Number of bytes filled in the queue + - name: filled.events + type: long + metric_type: gauge + description: > + Number of events filled in the queue + - name: filled.pct + type: float + metric_type: gauge + description: > + Percentage of the queue filled + - name: max_events + type: long + metric_type: gauge + description: > + Maximum number of events allowed in the queue + - name: removed.bytes + type: long + metric_type: counter + description: > + Number of bytes removed from the queue + - name: removed.events + type: long + metric_type: counter + description: > + Number of events removed from the queue - name: events type: group fields: @@ -665,6 +719,22 @@ type: long description: > Number of write errors + - name: latency + type: group + fields: + - name: histogram + type: group + fields: + - name: count + type: long + - name: max + type: float + - name: median + type: long + - name: p95 + type: float + - name: p99 + type: float - name: output type: group fields: diff --git a/metricbeat/module/beat/stats/data.go b/metricbeat/module/beat/stats/data.go index 8cb118669690..b6500903c9e7 100644 --- a/metricbeat/module/beat/stats/data.go +++ b/metricbeat/module/beat/stats/data.go @@ -72,10 +72,13 @@ var ( "bytes": c.Int("bytes"), "errors": c.Int("errors"), "latency": c.Dict("latency", s.Schema{ - "count": c.Int("count"), - "max": c.Int("max"), - "median": c.Float("median"), - "p99": c.Float("p99"), + "histogram": c.Dict("histogram", s.Schema{ + "count": c.Int("count"), + "max": c.Int("max"), + "median": c.Float("median"), + "p95": c.Float("p95"), + "p99": c.Float("p99"), + }), }), }), }),