Fix cisagov#509

I discovered that there are some conflicts between ECS's DNS fields (https://www.elastic.co/guide/en/ecs/current/ecs-dns.html) and Arkime's (https://github.com/arkime/arkime/blob/70765f46f6e17b62e405d9cd82d8109030e51bd8/db/db.pl#L4369-L4431) that would result in some issues when opening Zeek dns.log entries in Arkime sessions. So I'm commenting-out some of the ECS DNS normalization here in favor of the Arkime fields.
mmguero-dev · Nov 22, 2024 · 8cb4573 · 8cb4573
1 parent 6a6cc49
commit 8cb4573
Showing 1 changed file with 22 additions and 15 deletions.
diff --git a/logstash/pipelines/zeek/1200_zeek_mutate.conf b/logstash/pipelines/zeek/1200_zeek_mutate.conf
@@ -440,6 +440,13 @@ filter {
   } else if ([log_source] == "dns") {
     #############################################################################################################################
     # dns.log specific logic
+    #
+    # NOTE: I discovered that there are some conflicts between ECS's DNS fields
+    #   (https://www.elastic.co/guide/en/ecs/current/ecs-dns.html) and Arkime's
+    #   (https://github.com/arkime/arkime/blob/70765f46f6e17b62e405d9cd82d8109030e51bd8/db/db.pl#L4369-L4431)
+    #   that would result in some issues when opening Arkime sessions with DNS records
+    #   (see https://github.com/cisagov/malcolm/issues/509). So I'm commenting-out some of the ECS DNS
+    #   normalization here in favor of the Arkime fields.
 
     # this must be done because [dns][ip] only handles IPv4 or IPv6 addresses
     # but [answers] can contain hostnames as well
@@ -456,11 +463,11 @@ filter {
         "
       }
       # ECS - zeek.dns -> dns.answers
-      ruby {
-        id => "ruby_zeek_dns_answers_to_ecs"
-        code => '
-          event.set("[dns][answers]", [Array(event.get("[zeek][dns][answers]")), Array(event.get("[zeek][dns][TTLs]"))].transpose.map{ |d| Hash[[:data, :ttl].zip(d)] })
-      '}
+      # ruby {
+      #   id => "ruby_zeek_dns_answers_to_ecs"
+      #   code => '
+      #     event.set("[dns][answers]", [Array(event.get("[zeek][dns][answers]")), Array(event.get("[zeek][dns][TTLs]"))].transpose.map{ |d| Hash[[:data, :ttl].zip(d)] })
+      # '}
       mutate { id => "mutate_add_field_ecs_dns_type_answer"
                add_field => { "[dns][type]" => "answer" } }
     }
@@ -491,21 +498,21 @@ filter {
     }
 
     # dns.op_code
-    if ([dns][opcode]) {                          mutate { id => "mutate_add_field_ecs_dns_opcode"
-                                                           add_field => { "[dns][op_code]" => "%{[dns][opcode]}" } } }
+   if ([dns][opcode]) {                          mutate { id => "mutate_add_field_ecs_dns_opcode"
+                                                          add_field => { "[dns][op_code]" => "%{[dns][opcode]}" } } }
 
     # dns.question.class
     if ([zeek][dns][qclass_name]) {                mutate { id => "mutate_add_field_ecs_dns_qclass"
                                                            add_field => { "[dns][question][class]" => "%{[zeek][dns][qclass_name]}" } } }
 
     # dns.question.type
-    if ([zeek][dns][qtype_name]) {                 mutate { id => "mutate_add_field_ecs_dns_qtype"
-                                                           add_field => { "[dns][question][type]" => "%{[zeek][dns][qtype_name]}" } } }
+    # if ([zeek][dns][qtype_name]) {                 mutate { id => "mutate_add_field_ecs_dns_qtype"
+    #                                                         add_field => { "[dns][question][type]" => "%{[zeek][dns][qtype_name]}" } } }
 
     # dns.question.name and dns.type:query
     if ([zeek][dns][query]) {
-                          mutate { id => "mutate_add_field_ecs_dns_query"
-                                   add_field => { "[dns][question][name]" => "%{[zeek][dns][query]}" } }
+      # mutate { id => "mutate_add_field_ecs_dns_query"
+      #          add_field => { "[dns][question][name]" => "%{[zeek][dns][query]}" } }
       if (![dns][type]) { mutate { id => "mutate_add_field_ecs_dns_type_query"
                                    add_field => { "[dns][type]" => "query" } } }
     }
@@ -523,12 +530,12 @@ filter {
     }
 
     # dns.response_code
-    if ([zeek][dns][rcode_name]) {                 mutate { id => "mutate_add_field_ecs_dns_response_code"
-                                                           add_field => { "[dns][response_code]" => "%{[zeek][dns][rcode_name]}" } } }
+    # if ([zeek][dns][rcode_name]) {                mutate { id => "mutate_add_field_ecs_dns_response_code"
+    #                                                        add_field => { "[dns][response_code]" => "%{[zeek][dns][rcode_name]}" } } }
 
     # dns.id
-    if ([zeek][dns][trans_id]) {                   mutate { id => "mutate_add_field_ecs_dns_id"
-                                                           add_field => { "[dns][id]" => "%{[zeek][dns][trans_id]}" } } }
+    # if ([zeek][dns][trans_id]) {                   mutate { id => "mutate_add_field_ecs_dns_id"
+    #                                                         add_field => { "[dns][id]" => "%{[zeek][dns][trans_id]}" } } }
 
     # TODO: domain stuff (dns.question.registered_domain, dns.question.subdomain, dns.question.top_level_domain)
     # perhaps use something like https://github.com/plutonbacon/logstash-filter-publicsuffix