azure_frontdoor: fix types (#11272)

Convert timeTaken and timeToFirstByte to doubles representing seconds[1]. Remove redundant field definitions. [1]https://learn.microsoft.com/en-us/azure/frontdoor/front-door-diagnostics?pivots=front-door-standard-premium#access-log
elastic · Oct 1, 2024 · 27f35c8 · 27f35c8
1 parent 9099139
commit 27f35c8
Show file tree

Hide file tree

Showing 8 changed files with 44 additions and 29 deletions.
diff --git a/packages/azure_frontdoor/changelog.yml b/packages/azure_frontdoor/changelog.yml
@@ -1,3 +1,8 @@
+- version: "2.0.0"
+  changes:
+    - description: "Changed keyword field type to double in time_to_first_byte, time_taken fields and to date in time fields"
+      type: breaking-change
+      link: https://github.com/elastic/integrations/pull/11272
 - version: "1.10.0"
   changes:
     - description: "Allow @custom pipeline access to event.original without setting preserve_original_event."

diff --git a/...ges/azure_frontdoor/data_stream/access/_dev/test/pipeline/test-fdaccess.log-expected.json b/...ges/azure_frontdoor/data_stream/access/_dev/test/pipeline/test-fdaccess.log-expected.json
@@ -12,8 +12,8 @@
                         "pop": "SIN",
                         "routing_rule_name": "erp",
                         "rules_engine_match_names": [],
-                        "time_taken": "0.384",
-                        "time_to_first_byte": "0.384"
+                        "time_taken": 0.384,
+                        "time_to_first_byte": 0.384
                     },
                     "category": "FrontdoorAccessLog",
                     "operation_name": "Microsoft.Network/FrontDoor/AccessLog/Write",
@@ -117,8 +117,8 @@
                         "pop": "SIN",
                         "routing_rule_name": "erp",
                         "rules_engine_match_names": [],
-                        "time_taken": "0.122",
-                        "time_to_first_byte": "0.122"
+                        "time_taken": 0.122,
+                        "time_to_first_byte": 0.122
                     },
                     "category": "FrontdoorAccessLog",
                     "operation_name": "Microsoft.Network/FrontDoor/AccessLog/Write",
@@ -216,8 +216,8 @@
                         "pop": "SIN",
                         "routing_rule_name": "erp",
                         "rules_engine_match_names": [],
-                        "time_taken": "0.064",
-                        "time_to_first_byte": "0.064"
+                        "time_taken": 0.064,
+                        "time_to_first_byte": 0.064
                     },
                     "category": "FrontdoorAccessLog",
                     "operation_name": "Microsoft.Network/FrontDoor/AccessLog/Write",
@@ -316,8 +316,8 @@
                         "pop": "SIN",
                         "routing_rule_name": "erp",
                         "rules_engine_match_names": [],
-                        "time_taken": "0.064",
-                        "time_to_first_byte": "0.064"
+                        "time_taken": 0.064,
+                        "time_to_first_byte": 0.064
                     },
                     "category": "FrontdoorAccessLog",
                     "operation_name": "Microsoft.Network/FrontDoor/AccessLog/Write",
@@ -453,8 +453,8 @@
                         "pop": "SIN",
                         "routing_rule_name": "erp",
                         "rules_engine_match_names": [],
-                        "time_taken": "0.064",
-                        "time_to_first_byte": "0.064"
+                        "time_taken": 0.064,
+                        "time_to_first_byte": 0.064
                     },
                     "category": "FrontdoorAccessLog",
                     "operation_name": "Microsoft.Network/FrontDoor/AccessLog/Write",
@@ -615,8 +615,8 @@
                         "pop": "SIN",
                         "routing_rule_name": "erp",
                         "rules_engine_match_names": [],
-                        "time_taken": "0.064",
-                        "time_to_first_byte": "0.064"
+                        "time_taken": 0.064,
+                        "time_to_first_byte": 0.064
                     },
                     "category": "FrontdoorAccessLog",
                     "operation_name": "Microsoft.Network/FrontDoor/AccessLog/Write",

diff --git a/packages/azure_frontdoor/data_stream/access/elasticsearch/ingest_pipeline/default.yml b/packages/azure_frontdoor/data_stream/access/elasticsearch/ingest_pipeline/default.yml
@@ -76,9 +76,18 @@ processors:
       field: azure.frontdoor.access.properties.httpMethod
       target_field: http.request.method
       ignore_missing: true
-  - rename:
+  - convert:
       field: azure.frontdoor.access.properties.timeToFirstByte
       target_field: azure.frontdoor.access.time_to_first_byte
+      type: double
+      tag: convert_properties_time_to_first_byte
+      on_failure:
+        - append:
+            field: error.message
+            value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}'
+      ignore_missing: true
+  - remove:
+      field: azure.frontdoor.access.properties.timeToFirstByte
       ignore_missing: true
   - rename:
       field: azure.frontdoor.access.properties.pop
@@ -88,9 +97,18 @@ processors:
       field: azure.frontdoor.access.properties.responseBytes
       target_field: http.response.bytes
       ignore_missing: true
-  - rename:
+  - convert:
       field: azure.frontdoor.access.properties.timeTaken
       target_field: azure.frontdoor.access.time_taken
+      type: double
+      tag: convert_properties_time_taken
+      on_failure:
+        - append:
+            field: error.message
+            value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}'
+      ignore_missing: true
+  - remove:
+      field: azure.frontdoor.access.properties.timeTaken
       ignore_missing: true
   - rename:
       field: azure.frontdoor.access.properties.routingRuleName

diff --git a/packages/azure_frontdoor/data_stream/access/fields/fields.yml b/packages/azure_frontdoor/data_stream/access/fields/fields.yml
@@ -17,14 +17,11 @@
       type: group
       fields:
         - name: time_to_first_byte
-          type: keyword
+          type: double
           description: The length of time in milliseconds from AFD receives the request to the time the first byte gets sent to client, as measured on Azure Front Door. This property doesn't measure the client data.
         - name: time_taken
-          type: keyword
+          type: double
           description: The length of time from the time AFD edge server receives a client's request to the time that AFD sends the last byte of response to client, in milliseconds. This field doesn't take into account network latency and TCP buffering.
-        - name: time
-          type: keyword
-          description: The date and time when the AFD edge delivered requested contents to client (in UTC).
         - name: rules_engine_match_names
           type: keyword
           description: The names of the rules that were processed.

diff --git a/packages/azure_frontdoor/data_stream/access/sample_event.json b/packages/azure_frontdoor/data_stream/access/sample_event.json
@@ -17,8 +17,8 @@
                 "pop": "SGE",
                 "routing_rule_name": "erp",
                 "rules_engine_match_names": [],
-                "time_taken": "3.603",
-                "time_to_first_byte": "3.603"
+                "time_taken": 3.603,
+                "time_to_first_byte": 3.603
             },
             "category": "FrontdoorAccessLog",
             "operation_name": "Microsoft.Network/FrontDoor/AccessLog/Write",

diff --git a/packages/azure_frontdoor/data_stream/waf/fields/fields.yml b/packages/azure_frontdoor/data_stream/waf/fields/fields.yml
@@ -16,9 +16,6 @@
     - name: waf
       type: group
       fields:
-        - name: time
-          type: keyword
-          description: The date and time when the AFD edge delivered requested contents to client (in UTC).
         - name: policy
           type: keyword
           description: WAF policy name.

diff --git a/packages/azure_frontdoor/docs/README.md b/packages/azure_frontdoor/docs/README.md
@@ -81,9 +81,8 @@ Users can also use this in case of a Hybrid Cloud model, where one may define th
 | azure.frontdoor.access.pop | The edge pop, which responded to the user request. | keyword |
 | azure.frontdoor.access.routing_rule_name | The name of the route that the request matched. | keyword |
 | azure.frontdoor.access.rules_engine_match_names | The names of the rules that were processed. | keyword |
-| azure.frontdoor.access.time | The date and time when the AFD edge delivered requested contents to client (in UTC). | keyword |
-| azure.frontdoor.access.time_taken | The length of time from the time AFD edge server receives a client's request to the time that AFD sends the last byte of response to client, in milliseconds. This field doesn't take into account network latency and TCP buffering. | keyword |
-| azure.frontdoor.access.time_to_first_byte | The length of time in milliseconds from AFD receives the request to the time the first byte gets sent to client, as measured on Azure Front Door. This property doesn't measure the client data. | keyword |
+| azure.frontdoor.access.time_taken | The length of time from the time AFD edge server receives a client's request to the time that AFD sends the last byte of response to client, in milliseconds. This field doesn't take into account network latency and TCP buffering. | double |
+| azure.frontdoor.access.time_to_first_byte | The length of time in milliseconds from AFD receives the request to the time the first byte gets sent to client, as measured on Azure Front Door. This property doesn't measure the client data. | double |
 | azure.frontdoor.category | Azure frontdoor category name. | keyword |
 | azure.frontdoor.operation_name | Azure operation name. | keyword |
 | azure.frontdoor.resource_id | Azure Resource ID. | keyword |
@@ -129,7 +128,6 @@ Users can also use this in case of a Hybrid Cloud model, where one may define th
 | azure.frontdoor.waf.identity_name | identity name | keyword |
 | azure.frontdoor.waf.policy | WAF policy name. | keyword |
 | azure.frontdoor.waf.policy_mode | WAF policy mode. | keyword |
-| azure.frontdoor.waf.time | The date and time when the AFD edge delivered requested contents to client (in UTC). | keyword |
 | cloud.image.id | Image ID for the cloud instance. | keyword |
 | data_stream.dataset | Data stream dataset. | constant_keyword |
 | data_stream.namespace | Data stream namespace. | constant_keyword |

diff --git a/packages/azure_frontdoor/manifest.yml b/packages/azure_frontdoor/manifest.yml
@@ -1,7 +1,7 @@
 format_version: "3.0.2"
 name: azure_frontdoor
 title: "Azure Frontdoor"
-version: "1.10.0"
+version: "2.0.0"
 description: "This Elastic integration collects logs from Azure Frontdoor."
 type: integration
 categories: