From dc327bae7405c645e7a046773d21e44f04b5ebd9 Mon Sep 17 00:00:00 2001 From: Dave Marco Date: Fri, 7 Mar 2025 17:31:36 +0000 Subject: [PATCH 01/15] latest --- .../user-guide/format-struct-logs-syntax.md | 41 ++++++++++++++++--- docs/src/user-guide/log-level-filtering.md | 8 +++- 2 files changed, 41 insertions(+), 8 deletions(-) diff --git a/docs/src/user-guide/format-struct-logs-syntax.md b/docs/src/user-guide/format-struct-logs-syntax.md index 29ae306b2..7bdd995bf 100644 --- a/docs/src/user-guide/format-struct-logs-syntax.md +++ b/docs/src/user-guide/format-struct-logs-syntax.md @@ -24,10 +24,12 @@ Defines the key of the field whose value should replace the placeholder. * Nested fields can be specified using periods (`.`) to denote hierarchy. * E.g., the field `{"a:" {"b": 0}}` may be denoted by `a.b`. +* CLP IR auto-generated keys can specified by adding a `@` prefix to the field name. + * E.g., the auto-generated field `timestamp` may be denoted by `@timestamp`. * Field names can contain any character, except the following characters must be escaped with a backslash: * `.` - * `$` + * `@` * `{` * `}` * `:` @@ -63,10 +65,12 @@ Every format string contains an implicit trailing newline so that each formatted a newline. ## Examples -Consider the following log event: + +### Example 1 +Consider the following JSON log event: ``` { - "@timestamp": 1427153388942, + "ts": 1427153388942, "level": "INFO", "thread": 0, "latency": { @@ -80,10 +84,10 @@ Consider the following log event: We can format this using the following YScope format string: ``` -{@timestamp:timestamp:YYYY-MM-DD HH\:mm\:ss.SSS} {level} \{{thread}\} latency={latency.secs:round} {an\.odd\.key\{name\}} +{ts:timestamp:YYYY-MM-DD HH\:mm\:ss.SSS} {level} \{{thread}\} latency={latency.secs:round} {an\.odd\.key\{name\}} ``` -* In the first placeholder, we have the field name `@timestamp`, a formatter called `timestamp`, and +* In the first placeholder, we have the field name `ts`, a formatter called `timestamp`, and the formatter's options which are a date format string. * The second and third placeholders simply stringify the values of the given fields. * The fourth placeholder uses the `round` formatter to round a nested field's value; this @@ -94,5 +98,30 @@ The formatted string will be: ``` 2015-03-23 19:29:48.942 INFO {0} latency=56 org.apache.hadoop.metrics2.impl.MetricsConfig: loaded properties from hadoop-metrics2.properties ``` +### Example 2 + +Consider the following Key-Value Pair CLP IR log event: +``` +{ + "auto-generated": { + "ts": 1732733160216, + "level": "INFO", + }, + "user-generated": { + "message": "Accepted socket connection from /192.168.1.100:50002" + }, +} +``` + +We can format this using the following YScope format string: + +``` +{@ts:timestamp} {@level} {message} +``` + +The formatted string will be: +``` +2024-11-27T18:46:00Z INFO Accepted socket connection from /192.168.1.100:50002 +``` -For a list of currently supported formatters, see [Formatters](format-struct-logs-formatters). +For a list of currently supported formatters, see [Formatters](format-struct-logs-formatters). \ No newline at end of file diff --git a/docs/src/user-guide/log-level-filtering.md b/docs/src/user-guide/log-level-filtering.md index a79e61464..ab11b59a8 100644 --- a/docs/src/user-guide/log-level-filtering.md +++ b/docs/src/user-guide/log-level-filtering.md @@ -1,6 +1,10 @@ # Overview -The log viewer offers two ways to filter logs by their level. +The log viewer offers two ways to filter logs by their level. You can enable filtering +by entering the key containing the log level in the settings ({fas}`gear`) dialog. +The syntax for the key is the same as the format string [field name syntax](format-struct-logs-syntax). + +* [Format string syntax](format-struct-logs-syntax) ## Severity-based filtering You can filter for log events at or above a selected level by clicking the level's name. For @@ -12,4 +16,4 @@ example, selecting INFO shows INFO, WARN, ERROR, and FATAL logs. You can select specific log levels to show or hide by checking or unchecking the checkbox next to each level's name. -![Individual level filtering](individual-filter.png) +![Individual level filtering](individual-filter.png) \ No newline at end of file From 84cf8d24f4740c1d304e265aacf3510b4459f71a Mon Sep 17 00:00:00 2001 From: Dave Marco Date: Fri, 7 Mar 2025 18:52:43 +0000 Subject: [PATCH 02/15] latest --- .../user-guide/format-struct-logs-syntax.md | 31 ++++++++++++------- docs/src/user-guide/log-level-filtering.md | 8 ++--- 2 files changed, 24 insertions(+), 15 deletions(-) diff --git a/docs/src/user-guide/format-struct-logs-syntax.md b/docs/src/user-guide/format-struct-logs-syntax.md index 7bdd995bf..d0f209562 100644 --- a/docs/src/user-guide/format-struct-logs-syntax.md +++ b/docs/src/user-guide/format-struct-logs-syntax.md @@ -24,8 +24,10 @@ Defines the key of the field whose value should replace the placeholder. * Nested fields can be specified using periods (`.`) to denote hierarchy. * E.g., the field `{"a:" {"b": 0}}` may be denoted by `a.b`. -* CLP IR auto-generated keys can specified by adding a `@` prefix to the field name. - * E.g., the auto-generated field `timestamp` may be denoted by `@timestamp`. +* Auto-generated fields in a +[Key-Value Pair IR Stream](https://docs.yscope.com/clp/main/dev-guide/design-key-value-pair-ir-stream.html) +can be specified using `@` as a prefix. + * E.g., the auto-generated field `ts` may be denoted by `@ts`. * Field names can contain any character, except the following characters must be escaped with a backslash: * `.` @@ -66,7 +68,8 @@ a newline. ## Examples -### Example 1 +### JSON Example + Consider the following JSON log event: ``` { @@ -98,30 +101,36 @@ The formatted string will be: ``` 2015-03-23 19:29:48.942 INFO {0} latency=56 org.apache.hadoop.metrics2.impl.MetricsConfig: loaded properties from hadoop-metrics2.properties ``` -### Example 2 -Consider the following Key-Value Pair CLP IR log event: +### Key-Value Pair IR Example + +Consider the following Key-Value Pair IR log event: ``` { "auto-generated": { - "ts": 1732733160216, - "level": "INFO", + "ts": 1741371422000 }, "user-generated": { - "message": "Accepted socket connection from /192.168.1.100:50002" - }, + "message": "Callback registered to fire in 5 seconds:", + "ts": 1741371427000 + } } ``` We can format this using the following YScope format string: ``` -{@ts:timestamp} {@level} {message} +{@ts:timestamp} {message} {ts:timestamp} ``` +* In the first placeholder, we have the auto-generated field name `@ts`. The `@` prefix + specifies the field is from the auto-generated namespace. +* The second and third placeholders refer to the message and timestamp fields in the + user-generated namespace, respectively. + The formatted string will be: ``` -2024-11-27T18:46:00Z INFO Accepted socket connection from /192.168.1.100:50002 +2025-03-07T18:17:02Z Callback registered to fire in 5 seconds: 2025-03-07T18:17:07Z ``` For a list of currently supported formatters, see [Formatters](format-struct-logs-formatters). \ No newline at end of file diff --git a/docs/src/user-guide/log-level-filtering.md b/docs/src/user-guide/log-level-filtering.md index ab11b59a8..2acc17eb5 100644 --- a/docs/src/user-guide/log-level-filtering.md +++ b/docs/src/user-guide/log-level-filtering.md @@ -1,10 +1,10 @@ # Overview -The log viewer offers two ways to filter logs by their level. You can enable filtering -by entering the key containing the log level in the settings ({fas}`gear`) dialog. -The syntax for the key is the same as the format string [field name syntax](format-struct-logs-syntax). +The log viewer offers two ways to filter logs by their level. -* [Format string syntax](format-struct-logs-syntax) +For JSON logs, you can enable filtering by entering the log level key in the settings ({fas}`gear`) +dialog. The syntax for the key is the same as the field name in the + [format string](format-struct-logs-syntax). ## Severity-based filtering You can filter for log events at or above a selected level by clicking the level's name. For From f6d327737a714b5babd35d4074ad7228a5620e60 Mon Sep 17 00:00:00 2001 From: Dave Marco Date: Fri, 7 Mar 2025 18:57:02 +0000 Subject: [PATCH 03/15] latest --- docs/src/user-guide/log-level-filtering.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/src/user-guide/log-level-filtering.md b/docs/src/user-guide/log-level-filtering.md index 2acc17eb5..ca52bd7bb 100644 --- a/docs/src/user-guide/log-level-filtering.md +++ b/docs/src/user-guide/log-level-filtering.md @@ -16,4 +16,4 @@ example, selecting INFO shows INFO, WARN, ERROR, and FATAL logs. You can select specific log levels to show or hide by checking or unchecking the checkbox next to each level's name. -![Individual level filtering](individual-filter.png) \ No newline at end of file +![Individual level filtering](individual-filter.png) From fedc491d8203ce0fd507c63215ffac06c663b154 Mon Sep 17 00:00:00 2001 From: Dave Marco Date: Fri, 7 Mar 2025 19:07:49 +0000 Subject: [PATCH 04/15] small change showing escaped \@ --- docs/src/user-guide/format-struct-logs-syntax.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/src/user-guide/format-struct-logs-syntax.md b/docs/src/user-guide/format-struct-logs-syntax.md index d0f209562..0aa1dd5e3 100644 --- a/docs/src/user-guide/format-struct-logs-syntax.md +++ b/docs/src/user-guide/format-struct-logs-syntax.md @@ -80,14 +80,14 @@ Consider the following JSON log event: "msecs": 56400, "secs": 56.4, }, - "an.odd.key{name}": "org.apache.hadoop.metrics2.impl.MetricsConfig: loaded properties from hadoop-metrics2.properties" + "@an.odd.key{name}": "org.apache.hadoop.metrics2.impl.MetricsConfig: loaded properties from hadoop-metrics2.properties" } ``` We can format this using the following YScope format string: ``` -{ts:timestamp:YYYY-MM-DD HH\:mm\:ss.SSS} {level} \{{thread}\} latency={latency.secs:round} {an\.odd\.key\{name\}} +{ts:timestamp:YYYY-MM-DD HH\:mm\:ss.SSS} {level} \{{thread}\} latency={latency.secs:round} {\@an\.odd\.key\{name\}} ``` * In the first placeholder, we have the field name `ts`, a formatter called `timestamp`, and From efcfc4918a56cc64e5d7e9f9a5507b242a0d408c Mon Sep 17 00:00:00 2001 From: davemarco <83603688+davemarco@users.noreply.github.com> Date: Sun, 9 Mar 2025 21:07:52 -0400 Subject: [PATCH 05/15] Apply suggestions from code review Co-authored-by: kirkrodrigues <2454684+kirkrodrigues@users.noreply.github.com> --- .../user-guide/format-struct-logs-syntax.md | 29 ++++++++++++------- 1 file changed, 18 insertions(+), 11 deletions(-) diff --git a/docs/src/user-guide/format-struct-logs-syntax.md b/docs/src/user-guide/format-struct-logs-syntax.md index 0aa1dd5e3..67ad3a08c 100644 --- a/docs/src/user-guide/format-struct-logs-syntax.md +++ b/docs/src/user-guide/format-struct-logs-syntax.md @@ -26,8 +26,8 @@ Defines the key of the field whose value should replace the placeholder. * E.g., the field `{"a:" {"b": 0}}` may be denoted by `a.b`. * Auto-generated fields in a [Key-Value Pair IR Stream](https://docs.yscope.com/clp/main/dev-guide/design-key-value-pair-ir-stream.html) -can be specified using `@` as a prefix. - * E.g., the auto-generated field `ts` may be denoted by `@ts`. + can be specified by using `@` as a prefix. + * E.g., the auto-generated field `ts` would be specified as `@ts`. * Field names can contain any character, except the following characters must be escaped with a backslash: * `.` @@ -68,10 +68,10 @@ a newline. ## Examples -### JSON Example +### Formatting JSON logs events Consider the following JSON log event: -``` +```json { "ts": 1427153388942, "level": "INFO", @@ -102,10 +102,17 @@ The formatted string will be: 2015-03-23 19:29:48.942 INFO {0} latency=56 org.apache.hadoop.metrics2.impl.MetricsConfig: loaded properties from hadoop-metrics2.properties ``` -### Key-Value Pair IR Example +### Formatting kv-pair IR log events -Consider the following Key-Value Pair IR log event: -``` +Consider the following kv-pair IR log event: + +:::{note} +In the example below, for simplicity, we render the log event as JSON with the auto-generated +kv-pairs under the `auto-generated` key, and the user-generated kv-pairs under the `user-generated` +key, but these keys don't exist in the log event. +::: + +```json { "auto-generated": { "ts": 1741371422000 @@ -123,10 +130,10 @@ We can format this using the following YScope format string: {@ts:timestamp} {message} {ts:timestamp} ``` -* In the first placeholder, we have the auto-generated field name `@ts`. The `@` prefix - specifies the field is from the auto-generated namespace. -* The second and third placeholders refer to the message and timestamp fields in the - user-generated namespace, respectively. +* In the first placeholder, we have the auto-generated field name `@ts`. The `@` prefix specifies + that the field is from the auto-generated namespace. +* The second and third placeholders refer to the `message` and `ts` fields in the user-generated + namespace. The formatted string will be: ``` From fbe3d67b58c3e400bf56700bfd0679d8dbd7bc7a Mon Sep 17 00:00:00 2001 From: Dave Marco Date: Mon, 10 Mar 2025 01:27:22 +0000 Subject: [PATCH 06/15] kirk review --- docs/src/user-guide/format-struct-logs-syntax.md | 13 +++++++------ 1 file changed, 7 insertions(+), 6 deletions(-) diff --git a/docs/src/user-guide/format-struct-logs-syntax.md b/docs/src/user-guide/format-struct-logs-syntax.md index 67ad3a08c..07fea2695 100644 --- a/docs/src/user-guide/format-struct-logs-syntax.md +++ b/docs/src/user-guide/format-struct-logs-syntax.md @@ -24,9 +24,8 @@ Defines the key of the field whose value should replace the placeholder. * Nested fields can be specified using periods (`.`) to denote hierarchy. * E.g., the field `{"a:" {"b": 0}}` may be denoted by `a.b`. -* Auto-generated fields in a -[Key-Value Pair IR Stream](https://docs.yscope.com/clp/main/dev-guide/design-key-value-pair-ir-stream.html) - can be specified by using `@` as a prefix. +* Auto-generated fields in a [Key-Value Pair IR Stream][kvPairIr] can be specified by using `@` as a +prefix. * E.g., the auto-generated field `ts` would be specified as `@ts`. * Field names can contain any character, except the following characters must be escaped with a backslash: @@ -47,6 +46,8 @@ The name of the formatter to apply to the value before inserting it into the str * `:` * `\` +For a list of currently supported formatters, see [Formatters](format-struct-logs-formatters). + ### formatter-options (optional) Defines any options for the formatter denoted by `formatter-name`. @@ -90,8 +91,8 @@ We can format this using the following YScope format string: {ts:timestamp:YYYY-MM-DD HH\:mm\:ss.SSS} {level} \{{thread}\} latency={latency.secs:round} {\@an\.odd\.key\{name\}} ``` -* In the first placeholder, we have the field name `ts`, a formatter called `timestamp`, and - the formatter's options which are a date format string. +* In the first placeholder, we have the field name `ts`, a formatter called `timestamp`, and the + formatter's options which are a date format string. * The second and third placeholders simply stringify the values of the given fields. * The fourth placeholder uses the `round` formatter to round a nested field's value; this placeholder doesn't specify any formatter options, so the defaults will be used. @@ -140,4 +141,4 @@ The formatted string will be: 2025-03-07T18:17:02Z Callback registered to fire in 5 seconds: 2025-03-07T18:17:07Z ``` -For a list of currently supported formatters, see [Formatters](format-struct-logs-formatters). \ No newline at end of file +[kvPairIr]: https://docs.yscope.com/clp/main/dev-guide/design-key-value-pair-ir-stream.html From 1ef7972952e1e838c517dc26b0ce79ecc8d299c4 Mon Sep 17 00:00:00 2001 From: davemarco <83603688+davemarco@users.noreply.github.com> Date: Mon, 10 Mar 2025 10:17:26 -0400 Subject: [PATCH 07/15] Apply suggestions from code review Co-authored-by: kirkrodrigues <2454684+kirkrodrigues@users.noreply.github.com> --- docs/src/user-guide/format-struct-logs-syntax.md | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/docs/src/user-guide/format-struct-logs-syntax.md b/docs/src/user-guide/format-struct-logs-syntax.md index 07fea2695..cf57c7cdb 100644 --- a/docs/src/user-guide/format-struct-logs-syntax.md +++ b/docs/src/user-guide/format-struct-logs-syntax.md @@ -24,8 +24,8 @@ Defines the key of the field whose value should replace the placeholder. * Nested fields can be specified using periods (`.`) to denote hierarchy. * E.g., the field `{"a:" {"b": 0}}` may be denoted by `a.b`. -* Auto-generated fields in a [Key-Value Pair IR Stream][kvPairIr] can be specified by using `@` as a -prefix. +* Auto-generated fields in a [Key-Value Pair IR Stream][kv-pair-ir] can be specified by using `@` as + a prefix. * E.g., the auto-generated field `ts` would be specified as `@ts`. * Field names can contain any character, except the following characters must be escaped with a backslash: @@ -92,7 +92,7 @@ We can format this using the following YScope format string: ``` * In the first placeholder, we have the field name `ts`, a formatter called `timestamp`, and the - formatter's options which are a date format string. + formatter's options which are a date format string. * The second and third placeholders simply stringify the values of the given fields. * The fourth placeholder uses the `round` formatter to round a nested field's value; this placeholder doesn't specify any formatter options, so the defaults will be used. @@ -141,4 +141,4 @@ The formatted string will be: 2025-03-07T18:17:02Z Callback registered to fire in 5 seconds: 2025-03-07T18:17:07Z ``` -[kvPairIr]: https://docs.yscope.com/clp/main/dev-guide/design-key-value-pair-ir-stream.html +[kv-pair-ir]: https://docs.yscope.com/clp/main/dev-guide/design-key-value-pair-ir-stream.html From 98999a404fb43a15292341cd167496a6cfbbcf09 Mon Sep 17 00:00:00 2001 From: Dave Marco Date: Tue, 11 Mar 2025 15:34:58 +0000 Subject: [PATCH 08/15] kirk review --- docs/src/user-guide/index.md | 12 +++--- docs/src/user-guide/log-level-filtering.md | 4 -- .../format/formatters.md} | 0 .../format/index.md} | 15 +++++-- .../format/syntax.md} | 28 ++++-------- docs/src/user-guide/struct-logs/index.md | 28 ++++++++++++ docs/src/user-guide/struct-logs/key-syntax.md | 15 +++++++ .../user-guide/struct-logs/specifying-keys.md | 11 +++++ .../modals/SettingsModal/SettingsDialog.tsx | 43 ++++++++++++++++--- 9 files changed, 116 insertions(+), 40 deletions(-) rename docs/src/user-guide/{format-struct-logs-formatters.md => struct-logs/format/formatters.md} (100%) rename docs/src/user-guide/{format-struct-logs-overview.md => struct-logs/format/index.md} (86%) rename docs/src/user-guide/{format-struct-logs-syntax.md => struct-logs/format/syntax.md} (79%) create mode 100644 docs/src/user-guide/struct-logs/index.md create mode 100644 docs/src/user-guide/struct-logs/key-syntax.md create mode 100644 docs/src/user-guide/struct-logs/specifying-keys.md diff --git a/docs/src/user-guide/index.md b/docs/src/user-guide/index.md index 7e0b909a7..f95d4a866 100644 --- a/docs/src/user-guide/index.md +++ b/docs/src/user-guide/index.md @@ -14,10 +14,10 @@ A guide to viewing log files in the log viewer. ::: :::{grid-item-card} -:link: format-struct-logs-overview -Formatting structured logs +:link: struct-logs/index +Working with structured logs ^^^ -A guide to formatting structured (e.g. JSON) logs as plain text. +A guide to working with structured logs. ::: :::{grid-item-card} @@ -37,12 +37,10 @@ quick-start :::{toctree} :hidden: -:caption: Formatting structured logs +:caption: Working with structured logs :glob: -format-struct-logs-overview -format-struct-logs-syntax -format-struct-logs-formatters +struct-logs/index ::: :::{toctree} diff --git a/docs/src/user-guide/log-level-filtering.md b/docs/src/user-guide/log-level-filtering.md index ca52bd7bb..a79e61464 100644 --- a/docs/src/user-guide/log-level-filtering.md +++ b/docs/src/user-guide/log-level-filtering.md @@ -2,10 +2,6 @@ The log viewer offers two ways to filter logs by their level. -For JSON logs, you can enable filtering by entering the log level key in the settings ({fas}`gear`) -dialog. The syntax for the key is the same as the field name in the - [format string](format-struct-logs-syntax). - ## Severity-based filtering You can filter for log events at or above a selected level by clicking the level's name. For example, selecting INFO shows INFO, WARN, ERROR, and FATAL logs. diff --git a/docs/src/user-guide/format-struct-logs-formatters.md b/docs/src/user-guide/struct-logs/format/formatters.md similarity index 100% rename from docs/src/user-guide/format-struct-logs-formatters.md rename to docs/src/user-guide/struct-logs/format/formatters.md diff --git a/docs/src/user-guide/format-struct-logs-overview.md b/docs/src/user-guide/struct-logs/format/index.md similarity index 86% rename from docs/src/user-guide/format-struct-logs-overview.md rename to docs/src/user-guide/struct-logs/format/index.md index e4039c01f..1b58fe800 100644 --- a/docs/src/user-guide/format-struct-logs-overview.md +++ b/docs/src/user-guide/struct-logs/format/index.md @@ -1,4 +1,4 @@ -# Overview +# Formatting as plain text The log viewer can format structured (e.g. JSON) logs as plain text using a format string. The format string allows you to select which fields to include and how they should be formatted. You can @@ -33,5 +33,14 @@ The formatted log would appear as: ``` For reference docs, see: -* [Format string syntax](format-struct-logs-syntax) -* [Formatters](format-struct-logs-formatters) +* [Format string syntax](syntax) +* [Formatters](formatters) + + +:::{toctree} +:hidden: +:caption: Cluster setup + +syntax +formatters +::: \ No newline at end of file diff --git a/docs/src/user-guide/format-struct-logs-syntax.md b/docs/src/user-guide/struct-logs/format/syntax.md similarity index 79% rename from docs/src/user-guide/format-struct-logs-syntax.md rename to docs/src/user-guide/struct-logs/format/syntax.md index cf57c7cdb..cdb973c13 100644 --- a/docs/src/user-guide/format-struct-logs-syntax.md +++ b/docs/src/user-guide/struct-logs/format/syntax.md @@ -16,25 +16,15 @@ backslash: Each field placeholder is enclosed in braces (`{` and `}`) and has the following form, consisting of three components: ``` -{[:[:]]} +{[:[:]]} ``` -### field-name (required) +### field-key (required) Defines the key of the field whose value should replace the placeholder. -* Nested fields can be specified using periods (`.`) to denote hierarchy. - * E.g., the field `{"a:" {"b": 0}}` may be denoted by `a.b`. -* Auto-generated fields in a [Key-Value Pair IR Stream][kv-pair-ir] can be specified by using `@` as - a prefix. - * E.g., the auto-generated field `ts` would be specified as `@ts`. -* Field names can contain any character, except the following characters must be escaped with a - backslash: - * `.` - * `@` - * `{` - * `}` - * `:` - * `\` +```{include} ../key-syntax.md + +``` ### formatter-name (optional) The name of the formatter to apply to the value before inserting it into the string. @@ -46,7 +36,7 @@ The name of the formatter to apply to the value before inserting it into the str * `:` * `\` -For a list of currently supported formatters, see [Formatters](format-struct-logs-formatters). +For a list of currently supported formatters, see [Formatters](formatters). ### formatter-options (optional) Defines any options for the formatter denoted by `formatter-name`. @@ -91,7 +81,7 @@ We can format this using the following YScope format string: {ts:timestamp:YYYY-MM-DD HH\:mm\:ss.SSS} {level} \{{thread}\} latency={latency.secs:round} {\@an\.odd\.key\{name\}} ``` -* In the first placeholder, we have the field name `ts`, a formatter called `timestamp`, and the +* In the first placeholder, we have the field key `ts`, a formatter called `timestamp`, and the formatter's options which are a date format string. * The second and third placeholders simply stringify the values of the given fields. * The fourth placeholder uses the `round` formatter to round a nested field's value; this @@ -131,7 +121,7 @@ We can format this using the following YScope format string: {@ts:timestamp} {message} {ts:timestamp} ``` -* In the first placeholder, we have the auto-generated field name `@ts`. The `@` prefix specifies +* In the first placeholder, we have the auto-generated field key `@ts`. The `@` prefix specifies that the field is from the auto-generated namespace. * The second and third placeholders refer to the `message` and `ts` fields in the user-generated namespace. @@ -140,5 +130,3 @@ The formatted string will be: ``` 2025-03-07T18:17:02Z Callback registered to fire in 5 seconds: 2025-03-07T18:17:07Z ``` - -[kv-pair-ir]: https://docs.yscope.com/clp/main/dev-guide/design-key-value-pair-ir-stream.html diff --git a/docs/src/user-guide/struct-logs/index.md b/docs/src/user-guide/struct-logs/index.md new file mode 100644 index 000000000..d335bd2b6 --- /dev/null +++ b/docs/src/user-guide/struct-logs/index.md @@ -0,0 +1,28 @@ +# Structured log guides + +The following are guides for working with structured logs. + +::::{grid} 1 1 2 2 +:gutter: 2 + +:::{grid-item-card} +:link: format/index +Formatting as plain text +^^^ +A guide to formatting logs as plain text with a format string. +::: + +:::{grid-item-card} +:link: specifying-keys +Specifying keys +^^^ +A guide on how to specify keys in YScope syntax. +::: +:::: + +:::{toctree} +:hidden: + +format/index +specifying-keys +::: \ No newline at end of file diff --git a/docs/src/user-guide/struct-logs/key-syntax.md b/docs/src/user-guide/struct-logs/key-syntax.md new file mode 100644 index 000000000..ab1f18ef1 --- /dev/null +++ b/docs/src/user-guide/struct-logs/key-syntax.md @@ -0,0 +1,15 @@ +* Nested keys can be specified using periods (`.`) to denote hierarchy. + * E.g., the field `{"a:" {"b": 0}}` may be denoted by `a.b`. +* Auto-generated keys in a [Key-Value Pair IR Stream][kv-pair-ir] can be specified by using `@` as + a prefix. + * E.g., the auto-generated key `ts` would be specified as `@ts`. +* Keys can contain any character, except the following characters must be escaped with a + backslash: + * `.` + * `@` + * `{` + * `}` + * `:` + * `\` + +[kv-pair-ir]: https://docs.yscope.com/clp/main/dev-guide/design-key-value-pair-ir-stream.html \ No newline at end of file diff --git a/docs/src/user-guide/struct-logs/specifying-keys.md b/docs/src/user-guide/struct-logs/specifying-keys.md new file mode 100644 index 000000000..fa59cc60e --- /dev/null +++ b/docs/src/user-guide/struct-logs/specifying-keys.md @@ -0,0 +1,11 @@ +# Specifying keys + +Viewing structured logs requires the user to specify keys for: +* The format string +* Authoritative fields such as the log level and timestamp + +Both options are found the settings dialog ({fas}`gear`). See below for the key syntax. + +```{include} key-syntax.md + +``` \ No newline at end of file diff --git a/src/components/modals/SettingsModal/SettingsDialog.tsx b/src/components/modals/SettingsModal/SettingsDialog.tsx index be3667eb0..1fac099e1 100644 --- a/src/components/modals/SettingsModal/SettingsDialog.tsx +++ b/src/components/modals/SettingsModal/SettingsDialog.tsx @@ -43,18 +43,19 @@ const getConfigFormFields = () => [ { helperText: ( - [JSON] Format string for formatting a JSON log event as plain text. See the + [Structured] Format string for formatting a structured log event as plain text. + See {" "} - format string syntax docs + here {" "} - or leave this blank to display the entire log event. + for syntax or leave this blank to display the entire log event. ), initialValue: getConfig(CONFIG_KEY.DECODER_OPTIONS).formatString, @@ -63,14 +64,44 @@ const getConfigFormFields = () => [ type: "text", }, { - helperText: "[JSON] Key to extract the log level from.", + helperText: ( + + [Structured] Key to extract the log level from. See + {" "} + + here + + {" "} + for syntax. + + ), initialValue: getConfig(CONFIG_KEY.DECODER_OPTIONS).logLevelKey, label: "Decoder: Log level key", name: LOCAL_STORAGE_KEY.DECODER_OPTIONS_LOG_LEVEL_KEY, type: "text", }, { - helperText: "[JSON] Key to extract the log timestamp from.", + helperText: ( + + [Structured] Key to extract the timestamp from. See + {" "} + + here + + {" "} + for syntax. + + ), initialValue: getConfig(CONFIG_KEY.DECODER_OPTIONS).timestampKey, label: "Decoder: Timestamp key", name: LOCAL_STORAGE_KEY.DECODER_OPTIONS_TIMESTAMP_KEY, From e119e5f182af527b80c9472418385ca06406f9aa Mon Sep 17 00:00:00 2001 From: davemarco <83603688+davemarco@users.noreply.github.com> Date: Tue, 11 Mar 2025 16:58:17 -0400 Subject: [PATCH 09/15] Apply suggestions from code review Co-authored-by: kirkrodrigues <2454684+kirkrodrigues@users.noreply.github.com> --- docs/src/user-guide/index.md | 2 +- docs/src/user-guide/struct-logs/format/index.md | 4 +--- docs/src/user-guide/struct-logs/format/syntax.md | 9 ++++----- docs/src/user-guide/struct-logs/key-syntax.md | 5 ++--- docs/src/user-guide/struct-logs/specifying-keys.md | 9 +++++---- src/components/modals/SettingsModal/SettingsDialog.tsx | 4 ++-- 6 files changed, 15 insertions(+), 18 deletions(-) diff --git a/docs/src/user-guide/index.md b/docs/src/user-guide/index.md index f95d4a866..877ea3c9d 100644 --- a/docs/src/user-guide/index.md +++ b/docs/src/user-guide/index.md @@ -17,7 +17,7 @@ A guide to viewing log files in the log viewer. :link: struct-logs/index Working with structured logs ^^^ -A guide to working with structured logs. +Guides and reference docs for working with structured logs. ::: :::{grid-item-card} diff --git a/docs/src/user-guide/struct-logs/format/index.md b/docs/src/user-guide/struct-logs/format/index.md index 1b58fe800..fcbc99c73 100644 --- a/docs/src/user-guide/struct-logs/format/index.md +++ b/docs/src/user-guide/struct-logs/format/index.md @@ -36,11 +36,9 @@ For reference docs, see: * [Format string syntax](syntax) * [Formatters](formatters) - :::{toctree} :hidden: -:caption: Cluster setup syntax formatters -::: \ No newline at end of file +::: diff --git a/docs/src/user-guide/struct-logs/format/syntax.md b/docs/src/user-guide/struct-logs/format/syntax.md index cdb973c13..795b804b1 100644 --- a/docs/src/user-guide/struct-logs/format/syntax.md +++ b/docs/src/user-guide/struct-logs/format/syntax.md @@ -22,9 +22,8 @@ three components: ### field-key (required) Defines the key of the field whose value should replace the placeholder. -```{include} ../key-syntax.md - -``` +:::{include} ../key-syntax.md +::: ### formatter-name (optional) The name of the formatter to apply to the value before inserting it into the string. @@ -69,7 +68,7 @@ Consider the following JSON log event: "thread": 0, "latency": { "msecs": 56400, - "secs": 56.4, + "secs": 56.4 }, "@an.odd.key{name}": "org.apache.hadoop.metrics2.impl.MetricsConfig: loaded properties from hadoop-metrics2.properties" } @@ -115,7 +114,7 @@ key, but these keys don't exist in the log event. } ``` -We can format this using the following YScope format string: +We can format this using the following format string: ``` {@ts:timestamp} {message} {ts:timestamp} diff --git a/docs/src/user-guide/struct-logs/key-syntax.md b/docs/src/user-guide/struct-logs/key-syntax.md index ab1f18ef1..5151968b5 100644 --- a/docs/src/user-guide/struct-logs/key-syntax.md +++ b/docs/src/user-guide/struct-logs/key-syntax.md @@ -3,8 +3,7 @@ * Auto-generated keys in a [Key-Value Pair IR Stream][kv-pair-ir] can be specified by using `@` as a prefix. * E.g., the auto-generated key `ts` would be specified as `@ts`. -* Keys can contain any character, except the following characters must be escaped with a - backslash: +* Keys can contain any character, except the following characters must be escaped with a backslash: * `.` * `@` * `{` @@ -12,4 +11,4 @@ * `:` * `\` -[kv-pair-ir]: https://docs.yscope.com/clp/main/dev-guide/design-key-value-pair-ir-stream.html \ No newline at end of file +[kv-pair-ir]: https://docs.yscope.com/clp/main/dev-guide/design-key-value-pair-ir-stream.html diff --git a/docs/src/user-guide/struct-logs/specifying-keys.md b/docs/src/user-guide/struct-logs/specifying-keys.md index fa59cc60e..8749a97ac 100644 --- a/docs/src/user-guide/struct-logs/specifying-keys.md +++ b/docs/src/user-guide/struct-logs/specifying-keys.md @@ -1,11 +1,12 @@ # Specifying keys -Viewing structured logs requires the user to specify keys for: +Viewing structured logs requires specifying keys for: * The format string * Authoritative fields such as the log level and timestamp -Both options are found the settings dialog ({fas}`gear`). See below for the key syntax. +Both options are found the settings dialog ({fas}`gear`). -```{include} key-syntax.md +## Syntax -``` \ No newline at end of file +:::{include} key-syntax.md +::: \ No newline at end of file diff --git a/src/components/modals/SettingsModal/SettingsDialog.tsx b/src/components/modals/SettingsModal/SettingsDialog.tsx index 1fac099e1..496d890fc 100644 --- a/src/components/modals/SettingsModal/SettingsDialog.tsx +++ b/src/components/modals/SettingsModal/SettingsDialog.tsx @@ -44,7 +44,7 @@ const getConfigFormFields = () => [ helperText: ( [Structured] Format string for formatting a structured log event as plain text. - See + Leave blank to display the entire log event. See {" "} [ here {" "} - for syntax or leave this blank to display the entire log event. + for syntax. ), initialValue: getConfig(CONFIG_KEY.DECODER_OPTIONS).formatString, From 92aaebdd2233a7768593065294d047f4991d11df Mon Sep 17 00:00:00 2001 From: Dave Marco Date: Tue, 11 Mar 2025 21:03:31 +0000 Subject: [PATCH 10/15] kirk review --- docs/src/user-guide/index.md | 2 ++ docs/src/user-guide/struct-logs/format/syntax.md | 2 +- docs/src/user-guide/struct-logs/index.md | 2 +- 3 files changed, 4 insertions(+), 2 deletions(-) diff --git a/docs/src/user-guide/index.md b/docs/src/user-guide/index.md index 877ea3c9d..55f71a6f1 100644 --- a/docs/src/user-guide/index.md +++ b/docs/src/user-guide/index.md @@ -41,6 +41,8 @@ quick-start :glob: struct-logs/index +struct-logs/format/index +struct-logs/specifying-keys ::: :::{toctree} diff --git a/docs/src/user-guide/struct-logs/format/syntax.md b/docs/src/user-guide/struct-logs/format/syntax.md index 795b804b1..453190558 100644 --- a/docs/src/user-guide/struct-logs/format/syntax.md +++ b/docs/src/user-guide/struct-logs/format/syntax.md @@ -74,7 +74,7 @@ Consider the following JSON log event: } ``` -We can format this using the following YScope format string: +We can format this using the following format string: ``` {ts:timestamp:YYYY-MM-DD HH\:mm\:ss.SSS} {level} \{{thread}\} latency={latency.secs:round} {\@an\.odd\.key\{name\}} diff --git a/docs/src/user-guide/struct-logs/index.md b/docs/src/user-guide/struct-logs/index.md index d335bd2b6..b234b1eeb 100644 --- a/docs/src/user-guide/struct-logs/index.md +++ b/docs/src/user-guide/struct-logs/index.md @@ -1,4 +1,4 @@ -# Structured log guides +# Overview The following are guides for working with structured logs. From f5bc74f1bc4b4dcabd44e07f890659917a003c9e Mon Sep 17 00:00:00 2001 From: Dave Marco Date: Tue, 11 Mar 2025 21:05:21 +0000 Subject: [PATCH 11/15] fix --- docs/src/user-guide/struct-logs/index.md | 7 ------- 1 file changed, 7 deletions(-) diff --git a/docs/src/user-guide/struct-logs/index.md b/docs/src/user-guide/struct-logs/index.md index b234b1eeb..a27a54a54 100644 --- a/docs/src/user-guide/struct-logs/index.md +++ b/docs/src/user-guide/struct-logs/index.md @@ -19,10 +19,3 @@ Specifying keys A guide on how to specify keys in YScope syntax. ::: :::: - -:::{toctree} -:hidden: - -format/index -specifying-keys -::: \ No newline at end of file From 50a59feb2a1a0242d4ce4b575902d224b2ec0159 Mon Sep 17 00:00:00 2001 From: Dave Marco Date: Tue, 11 Mar 2025 21:22:55 +0000 Subject: [PATCH 12/15] kirk review --- src/components/modals/SettingsModal/SettingsDialog.tsx | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/components/modals/SettingsModal/SettingsDialog.tsx b/src/components/modals/SettingsModal/SettingsDialog.tsx index 496d890fc..66c5e2d6c 100644 --- a/src/components/modals/SettingsModal/SettingsDialog.tsx +++ b/src/components/modals/SettingsModal/SettingsDialog.tsx @@ -66,7 +66,7 @@ const getConfigFormFields = () => [ { helperText: ( - [Structured] Key to extract the log level from. See + [Structured] Key that maps to each log event's log level. See {" "} [ { helperText: ( - [Structured] Key to extract the timestamp from. See + [Structured] Key that maps to each log event's timestamp. See {" "} Date: Tue, 11 Mar 2025 21:32:39 +0000 Subject: [PATCH 13/15] fix lint --- src/components/modals/SettingsModal/SettingsDialog.tsx | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/components/modals/SettingsModal/SettingsDialog.tsx b/src/components/modals/SettingsModal/SettingsDialog.tsx index 66c5e2d6c..488542a9c 100644 --- a/src/components/modals/SettingsModal/SettingsDialog.tsx +++ b/src/components/modals/SettingsModal/SettingsDialog.tsx @@ -66,7 +66,7 @@ const getConfigFormFields = () => [ { helperText: ( - [Structured] Key that maps to each log event's log level. See + [Structured] Key that maps to each log event's log level. See {" "} [ { helperText: ( - [Structured] Key that maps to each log event's timestamp. See + [Structured] Key that maps to each log event's timestamp. See {" "} Date: Tue, 11 Mar 2025 19:10:54 -0400 Subject: [PATCH 14/15] Apply suggestions from code review Co-authored-by: kirkrodrigues <2454684+kirkrodrigues@users.noreply.github.com> --- docs/src/user-guide/struct-logs/index.md | 6 +++--- src/components/modals/SettingsModal/SettingsDialog.tsx | 4 ++-- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/docs/src/user-guide/struct-logs/index.md b/docs/src/user-guide/struct-logs/index.md index a27a54a54..00d2f0a03 100644 --- a/docs/src/user-guide/struct-logs/index.md +++ b/docs/src/user-guide/struct-logs/index.md @@ -1,6 +1,6 @@ # Overview -The following are guides for working with structured logs. +The guides below describe how to configure the log viewer to work with structured logs. ::::{grid} 1 1 2 2 :gutter: 2 @@ -9,13 +9,13 @@ The following are guides for working with structured logs. :link: format/index Formatting as plain text ^^^ -A guide to formatting logs as plain text with a format string. +How to format structured logs as plain text, using a format string. ::: :::{grid-item-card} :link: specifying-keys Specifying keys ^^^ -A guide on how to specify keys in YScope syntax. +How to specify keys when configuring the log viewer. ::: :::: diff --git a/src/components/modals/SettingsModal/SettingsDialog.tsx b/src/components/modals/SettingsModal/SettingsDialog.tsx index 488542a9c..d2c7218cc 100644 --- a/src/components/modals/SettingsModal/SettingsDialog.tsx +++ b/src/components/modals/SettingsModal/SettingsDialog.tsx @@ -69,7 +69,7 @@ const getConfigFormFields = () => [ [Structured] Key that maps to each log event's log level. See {" "} [ [Structured] Key that maps to each log event's timestamp. See {" "} Date: Mon, 12 May 2025 17:27:23 -0400 Subject: [PATCH 15/15] Update docs/src/user-guide/struct-logs/key-syntax.md Co-authored-by: kirkrodrigues <2454684+kirkrodrigues@users.noreply.github.com> --- docs/src/user-guide/struct-logs/key-syntax.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/src/user-guide/struct-logs/key-syntax.md b/docs/src/user-guide/struct-logs/key-syntax.md index 5151968b5..0a2b7b5c4 100644 --- a/docs/src/user-guide/struct-logs/key-syntax.md +++ b/docs/src/user-guide/struct-logs/key-syntax.md @@ -11,4 +11,4 @@ * `:` * `\` -[kv-pair-ir]: https://docs.yscope.com/clp/main/dev-guide/design-key-value-pair-ir-stream.html +[kv-pair-ir]: https://docs.yscope.com/clp/main/dev-guide/design-kv-ir-streams/auto-gen-kv-pairs.html