From 27f35c8a0762a688f46a26fc5b5b0c5b23a713dc Mon Sep 17 00:00:00 2001 From: eyescreamxd Date: Tue, 1 Oct 2024 07:01:05 +0400 Subject: [PATCH] azure_frontdoor: fix types (#11272) Convert timeTaken and timeToFirstByte to doubles representing seconds[1]. Remove redundant field definitions. [1]https://learn.microsoft.com/en-us/azure/frontdoor/front-door-diagnostics?pivots=front-door-standard-premium#access-log --- packages/azure_frontdoor/changelog.yml | 5 ++++ .../pipeline/test-fdaccess.log-expected.json | 24 +++++++++---------- .../elasticsearch/ingest_pipeline/default.yml | 22 +++++++++++++++-- .../data_stream/access/fields/fields.yml | 7 ++---- .../data_stream/access/sample_event.json | 4 ++-- .../data_stream/waf/fields/fields.yml | 3 --- packages/azure_frontdoor/docs/README.md | 6 ++--- packages/azure_frontdoor/manifest.yml | 2 +- 8 files changed, 44 insertions(+), 29 deletions(-) diff --git a/packages/azure_frontdoor/changelog.yml b/packages/azure_frontdoor/changelog.yml index bb849e0dad27..f3a963614f78 100644 --- a/packages/azure_frontdoor/changelog.yml +++ b/packages/azure_frontdoor/changelog.yml @@ -1,3 +1,8 @@ +- version: "2.0.0" + changes: + - description: "Changed keyword field type to double in time_to_first_byte, time_taken fields and to date in time fields" + type: breaking-change + link: https://github.com/elastic/integrations/pull/11272 - version: "1.10.0" changes: - description: "Allow @custom pipeline access to event.original without setting preserve_original_event." diff --git a/packages/azure_frontdoor/data_stream/access/_dev/test/pipeline/test-fdaccess.log-expected.json b/packages/azure_frontdoor/data_stream/access/_dev/test/pipeline/test-fdaccess.log-expected.json index 6ad63fb013a3..377195084c28 100644 --- a/packages/azure_frontdoor/data_stream/access/_dev/test/pipeline/test-fdaccess.log-expected.json +++ b/packages/azure_frontdoor/data_stream/access/_dev/test/pipeline/test-fdaccess.log-expected.json @@ -12,8 +12,8 @@ "pop": "SIN", "routing_rule_name": "erp", "rules_engine_match_names": [], - "time_taken": "0.384", - "time_to_first_byte": "0.384" + "time_taken": 0.384, + "time_to_first_byte": 0.384 }, "category": "FrontdoorAccessLog", "operation_name": "Microsoft.Network/FrontDoor/AccessLog/Write", @@ -117,8 +117,8 @@ "pop": "SIN", "routing_rule_name": "erp", "rules_engine_match_names": [], - "time_taken": "0.122", - "time_to_first_byte": "0.122" + "time_taken": 0.122, + "time_to_first_byte": 0.122 }, "category": "FrontdoorAccessLog", "operation_name": "Microsoft.Network/FrontDoor/AccessLog/Write", @@ -216,8 +216,8 @@ "pop": "SIN", "routing_rule_name": "erp", "rules_engine_match_names": [], - "time_taken": "0.064", - "time_to_first_byte": "0.064" + "time_taken": 0.064, + "time_to_first_byte": 0.064 }, "category": "FrontdoorAccessLog", "operation_name": "Microsoft.Network/FrontDoor/AccessLog/Write", @@ -316,8 +316,8 @@ "pop": "SIN", "routing_rule_name": "erp", "rules_engine_match_names": [], - "time_taken": "0.064", - "time_to_first_byte": "0.064" + "time_taken": 0.064, + "time_to_first_byte": 0.064 }, "category": "FrontdoorAccessLog", "operation_name": "Microsoft.Network/FrontDoor/AccessLog/Write", @@ -453,8 +453,8 @@ "pop": "SIN", "routing_rule_name": "erp", "rules_engine_match_names": [], - "time_taken": "0.064", - "time_to_first_byte": "0.064" + "time_taken": 0.064, + "time_to_first_byte": 0.064 }, "category": "FrontdoorAccessLog", "operation_name": "Microsoft.Network/FrontDoor/AccessLog/Write", @@ -615,8 +615,8 @@ "pop": "SIN", "routing_rule_name": "erp", "rules_engine_match_names": [], - "time_taken": "0.064", - "time_to_first_byte": "0.064" + "time_taken": 0.064, + "time_to_first_byte": 0.064 }, "category": "FrontdoorAccessLog", "operation_name": "Microsoft.Network/FrontDoor/AccessLog/Write", diff --git a/packages/azure_frontdoor/data_stream/access/elasticsearch/ingest_pipeline/default.yml b/packages/azure_frontdoor/data_stream/access/elasticsearch/ingest_pipeline/default.yml index 9d07a6ddaa5c..36b5f38f1ea4 100644 --- a/packages/azure_frontdoor/data_stream/access/elasticsearch/ingest_pipeline/default.yml +++ b/packages/azure_frontdoor/data_stream/access/elasticsearch/ingest_pipeline/default.yml @@ -76,9 +76,18 @@ processors: field: azure.frontdoor.access.properties.httpMethod target_field: http.request.method ignore_missing: true - - rename: + - convert: field: azure.frontdoor.access.properties.timeToFirstByte target_field: azure.frontdoor.access.time_to_first_byte + type: double + tag: convert_properties_time_to_first_byte + on_failure: + - append: + field: error.message + value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' + ignore_missing: true + - remove: + field: azure.frontdoor.access.properties.timeToFirstByte ignore_missing: true - rename: field: azure.frontdoor.access.properties.pop @@ -88,9 +97,18 @@ processors: field: azure.frontdoor.access.properties.responseBytes target_field: http.response.bytes ignore_missing: true - - rename: + - convert: field: azure.frontdoor.access.properties.timeTaken target_field: azure.frontdoor.access.time_taken + type: double + tag: convert_properties_time_taken + on_failure: + - append: + field: error.message + value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' + ignore_missing: true + - remove: + field: azure.frontdoor.access.properties.timeTaken ignore_missing: true - rename: field: azure.frontdoor.access.properties.routingRuleName diff --git a/packages/azure_frontdoor/data_stream/access/fields/fields.yml b/packages/azure_frontdoor/data_stream/access/fields/fields.yml index 69887ad7341b..2a9532ad1c96 100644 --- a/packages/azure_frontdoor/data_stream/access/fields/fields.yml +++ b/packages/azure_frontdoor/data_stream/access/fields/fields.yml @@ -17,14 +17,11 @@ type: group fields: - name: time_to_first_byte - type: keyword + type: double description: The length of time in milliseconds from AFD receives the request to the time the first byte gets sent to client, as measured on Azure Front Door. This property doesn't measure the client data. - name: time_taken - type: keyword + type: double description: The length of time from the time AFD edge server receives a client's request to the time that AFD sends the last byte of response to client, in milliseconds. This field doesn't take into account network latency and TCP buffering. - - name: time - type: keyword - description: The date and time when the AFD edge delivered requested contents to client (in UTC). - name: rules_engine_match_names type: keyword description: The names of the rules that were processed. diff --git a/packages/azure_frontdoor/data_stream/access/sample_event.json b/packages/azure_frontdoor/data_stream/access/sample_event.json index 13f6bb8462fe..d8a295cc7308 100644 --- a/packages/azure_frontdoor/data_stream/access/sample_event.json +++ b/packages/azure_frontdoor/data_stream/access/sample_event.json @@ -17,8 +17,8 @@ "pop": "SGE", "routing_rule_name": "erp", "rules_engine_match_names": [], - "time_taken": "3.603", - "time_to_first_byte": "3.603" + "time_taken": 3.603, + "time_to_first_byte": 3.603 }, "category": "FrontdoorAccessLog", "operation_name": "Microsoft.Network/FrontDoor/AccessLog/Write", diff --git a/packages/azure_frontdoor/data_stream/waf/fields/fields.yml b/packages/azure_frontdoor/data_stream/waf/fields/fields.yml index 142672f658e8..db1d80d0f94d 100644 --- a/packages/azure_frontdoor/data_stream/waf/fields/fields.yml +++ b/packages/azure_frontdoor/data_stream/waf/fields/fields.yml @@ -16,9 +16,6 @@ - name: waf type: group fields: - - name: time - type: keyword - description: The date and time when the AFD edge delivered requested contents to client (in UTC). - name: policy type: keyword description: WAF policy name. diff --git a/packages/azure_frontdoor/docs/README.md b/packages/azure_frontdoor/docs/README.md index cd180df3ef7f..27cad82538ba 100644 --- a/packages/azure_frontdoor/docs/README.md +++ b/packages/azure_frontdoor/docs/README.md @@ -81,9 +81,8 @@ Users can also use this in case of a Hybrid Cloud model, where one may define th | azure.frontdoor.access.pop | The edge pop, which responded to the user request. | keyword | | azure.frontdoor.access.routing_rule_name | The name of the route that the request matched. | keyword | | azure.frontdoor.access.rules_engine_match_names | The names of the rules that were processed. | keyword | -| azure.frontdoor.access.time | The date and time when the AFD edge delivered requested contents to client (in UTC). | keyword | -| azure.frontdoor.access.time_taken | The length of time from the time AFD edge server receives a client's request to the time that AFD sends the last byte of response to client, in milliseconds. This field doesn't take into account network latency and TCP buffering. | keyword | -| azure.frontdoor.access.time_to_first_byte | The length of time in milliseconds from AFD receives the request to the time the first byte gets sent to client, as measured on Azure Front Door. This property doesn't measure the client data. | keyword | +| azure.frontdoor.access.time_taken | The length of time from the time AFD edge server receives a client's request to the time that AFD sends the last byte of response to client, in milliseconds. This field doesn't take into account network latency and TCP buffering. | double | +| azure.frontdoor.access.time_to_first_byte | The length of time in milliseconds from AFD receives the request to the time the first byte gets sent to client, as measured on Azure Front Door. This property doesn't measure the client data. | double | | azure.frontdoor.category | Azure frontdoor category name. | keyword | | azure.frontdoor.operation_name | Azure operation name. | keyword | | azure.frontdoor.resource_id | Azure Resource ID. | keyword | @@ -129,7 +128,6 @@ Users can also use this in case of a Hybrid Cloud model, where one may define th | azure.frontdoor.waf.identity_name | identity name | keyword | | azure.frontdoor.waf.policy | WAF policy name. | keyword | | azure.frontdoor.waf.policy_mode | WAF policy mode. | keyword | -| azure.frontdoor.waf.time | The date and time when the AFD edge delivered requested contents to client (in UTC). | keyword | | cloud.image.id | Image ID for the cloud instance. | keyword | | data_stream.dataset | Data stream dataset. | constant_keyword | | data_stream.namespace | Data stream namespace. | constant_keyword | diff --git a/packages/azure_frontdoor/manifest.yml b/packages/azure_frontdoor/manifest.yml index fdc94e03cbec..91277aa12454 100644 --- a/packages/azure_frontdoor/manifest.yml +++ b/packages/azure_frontdoor/manifest.yml @@ -1,7 +1,7 @@ format_version: "3.0.2" name: azure_frontdoor title: "Azure Frontdoor" -version: "1.10.0" +version: "2.0.0" description: "This Elastic integration collects logs from Azure Frontdoor." type: integration categories: