diff --git a/dashboards/dashboards/beats/4ca94c70-d7da-11ee-9ed3-e7afff29e59a.json b/dashboards/dashboards/beats/4ca94c70-d7da-11ee-9ed3-e7afff29e59a.json index 5bc781396..7a5bb0416 100644 --- a/dashboards/dashboards/beats/4ca94c70-d7da-11ee-9ed3-e7afff29e59a.json +++ b/dashboards/dashboards/beats/4ca94c70-d7da-11ee-9ed3-e7afff29e59a.json @@ -7,8 +7,8 @@ "namespaces": [ "default" ], - "updated_at": "2024-03-01T19:19:33.901Z", - "version": "WzEwOTksMV0=", + "updated_at": "2024-03-04T21:05:53.644Z", + "version": "Wzg4NCwxXQ==", "attributes": { "title": "Packet Capture Statistics", "hits": 0, @@ -108,8 +108,8 @@ "namespaces": [ "default" ], - "updated_at": "2024-03-01T19:12:21.755Z", - "version": "WzEwOTUsMV0=", + "updated_at": "2024-03-04T21:05:53.644Z", + "version": "Wzg4NSwxXQ==", "attributes": { "title": "Last Capture Metric Timestamp by Host", "visState": "{\"title\":\"Last Capture Metric Timestamp by Host\",\"type\":\"table\",\"aggs\":[{\"id\":\"1\",\"enabled\":true,\"type\":\"top_hits\",\"params\":{\"field\":\"@timestamp\",\"aggregate\":\"concat\",\"size\":1,\"sortField\":\"@timestamp\",\"sortOrder\":\"desc\",\"customLabel\":\"Last Metric Timestamp\"},\"schema\":\"metric\"},{\"id\":\"2\",\"enabled\":true,\"type\":\"terms\",\"params\":{\"field\":\"host.name\",\"orderBy\":\"_key\",\"order\":\"desc\",\"size\":100,\"otherBucket\":false,\"otherBucketLabel\":\"Other\",\"missingBucket\":false,\"missingBucketLabel\":\"Missing\",\"customLabel\":\"Capture Host\"},\"schema\":\"bucket\"},{\"id\":\"3\",\"enabled\":true,\"type\":\"terms\",\"params\":{\"field\":\"event.provider\",\"orderBy\":\"_key\",\"order\":\"desc\",\"size\":5,\"otherBucket\":true,\"otherBucketLabel\":\"Other\",\"missingBucket\":false,\"missingBucketLabel\":\"Missing\",\"customLabel\":\"Other\"},\"schema\":\"bucket\"}],\"params\":{\"perPage\":10,\"showPartialRows\":false,\"showMetricsAtAllLevels\":false,\"showTotal\":false,\"totalFunc\":\"sum\",\"percentageCol\":\"\"}}", @@ -137,11 +137,11 @@ "namespaces": [ "default" ], - "updated_at": "2024-03-01T18:50:17.404Z", - "version": "Wzk5NywxXQ==", + "updated_at": "2024-03-04T21:07:41.024Z", + "version": "Wzk5OSwxXQ==", "attributes": { "title": "Zeek and Suricata Capture Measurements ", - "visState": "{\"title\":\"Zeek and Suricata Capture Measurements \",\"type\":\"metrics\",\"aggs\":[],\"params\":{\"id\":\"61ca57f0-469d-11e7-af02-69e470af7417\",\"type\":\"timeseries\",\"series\":[{\"id\":\"32d1fca0-d7e1-11ee-ad81-217e54128a4b\",\"color\":\"rgba(33,150,243,1)\",\"split_mode\":\"everything\",\"metrics\":[{\"id\":\"32d1fca1-d7e1-11ee-ad81-217e54128a4b\",\"type\":\"sum\",\"field\":\"zeek.stats.pkts_link\"}],\"separate_axis\":0,\"axis_position\":\"right\",\"formatter\":\"number\",\"chart_type\":\"line\",\"line_width\":\"0\",\"point_size\":\"0\",\"fill\":0.5,\"stacked\":\"none\",\"label\":\"Zeek: packets seen\",\"type\":\"timeseries\",\"terms_field\":\"host.name\",\"terms_size\":\"25\",\"terms_order_by\":\"32d1fca1-d7e1-11ee-ad81-217e54128a4b\",\"split_color_mode\":\"opensearchDashboards\"},{\"id\":\"02bbf6a0-d7fb-11ee-a5f1-9ff9da698a18\",\"color\":\"rgba(84,179,153,1)\",\"split_mode\":\"everything\",\"metrics\":[{\"unit\":\"\",\"id\":\"02bbf6a1-d7fb-11ee-a5f1-9ff9da698a18\",\"type\":\"positive_rate\",\"field\":\"suricata.stats.capture.kernel_packets\"}],\"separate_axis\":0,\"axis_position\":\"right\",\"formatter\":\"number\",\"chart_type\":\"line\",\"line_width\":\"0\",\"point_size\":\"0\",\"fill\":0.5,\"stacked\":\"none\",\"label\":\"Suricata: packets seen\",\"type\":\"timeseries\"},{\"id\":\"e4143600-d7e0-11ee-ad81-217e54128a4b\",\"color\":\"rgba(229,115,115,1)\",\"split_mode\":\"everything\",\"metrics\":[{\"id\":\"e4143601-d7e0-11ee-ad81-217e54128a4b\",\"type\":\"sum\",\"field\":\"zeek.stats.pkts_dropped\"},{\"id\":\"f6df2790-d7e0-11ee-ad81-217e54128a4b\",\"type\":\"math\",\"variables\":[{\"id\":\"f8ee0a60-d7e0-11ee-ad81-217e54128a4b\",\"name\":\"packets\",\"field\":\"e4143601-d7e0-11ee-ad81-217e54128a4b\"}],\"script\":\"params.packets*-1\"}],\"separate_axis\":0,\"axis_position\":\"right\",\"formatter\":\"number\",\"chart_type\":\"line\",\"line_width\":\"0\",\"point_size\":\"0\",\"fill\":0.5,\"stacked\":\"none\",\"label\":\"Zeek: packets dropped\",\"type\":\"timeseries\",\"terms_field\":\"host.name\",\"terms_size\":\"25\",\"terms_order_by\":\"e4143601-d7e0-11ee-ad81-217e54128a4b\",\"split_color_mode\":\"opensearchDashboards\"},{\"id\":\"20b9a420-d7df-11ee-ad81-217e54128a4b\",\"color\":\"rgba(211,96,134,1)\",\"split_mode\":\"everything\",\"metrics\":[{\"id\":\"20b9a421-d7df-11ee-ad81-217e54128a4b\",\"type\":\"sum\",\"field\":\"zeek.capture_loss.gaps\"},{\"id\":\"9a3afce0-d7df-11ee-ad81-217e54128a4b\",\"type\":\"math\",\"variables\":[{\"id\":\"9dece150-d7df-11ee-ad81-217e54128a4b\",\"name\":\"gaps\",\"field\":\"20b9a421-d7df-11ee-ad81-217e54128a4b\"}],\"script\":\"params.gaps*-1\"}],\"separate_axis\":0,\"axis_position\":\"right\",\"formatter\":\"number\",\"chart_type\":\"line\",\"line_width\":\"0\",\"point_size\":\"0\",\"fill\":0.5,\"stacked\":\"none\",\"label\":\"Zeek: ACKS missed\",\"type\":\"timeseries\",\"terms_field\":\"host.name\",\"terms_size\":\"25\",\"terms_order_by\":\"20b9a421-d7df-11ee-ad81-217e54128a4b\",\"split_color_mode\":\"opensearchDashboards\"},{\"id\":\"cad40600-d7fb-11ee-a5f1-9ff9da698a18\",\"color\":\"rgba(255,171,145,1)\",\"split_mode\":\"everything\",\"metrics\":[{\"unit\":\"\",\"id\":\"cad40601-d7fb-11ee-a5f1-9ff9da698a18\",\"type\":\"positive_rate\",\"field\":\"suricata.stats.capture.kernel_drops\"},{\"id\":\"f5352cd0-d7fb-11ee-a5f1-9ff9da698a18\",\"type\":\"math\",\"variables\":[{\"id\":\"f79def70-d7fb-11ee-a5f1-9ff9da698a18\",\"name\":\"packets\",\"field\":\"cad40601-d7fb-11ee-a5f1-9ff9da698a18\"}],\"script\":\"params.packets*-1\"}],\"separate_axis\":0,\"axis_position\":\"right\",\"formatter\":\"number\",\"chart_type\":\"line\",\"line_width\":\"0\",\"point_size\":\"0\",\"fill\":0.5,\"stacked\":\"none\",\"label\":\"Suricata: packets dropped\",\"type\":\"timeseries\"}],\"time_field\":\"@timestamp\",\"index_pattern\":\"malcolm_beats_*\",\"interval\":\"\",\"axis_position\":\"left\",\"axis_formatter\":\"number\",\"axis_scale\":\"normal\",\"show_legend\":1,\"show_grid\":1,\"tooltip_mode\":\"show_all\",\"default_index_pattern\":\"arkime_sessions3-*\",\"default_timefield\":\"firstPacket\",\"isModelInvalid\":false,\"filter\":{\"query\":\"(event.provider:zeek OR event.provider:suricata) AND event.kind:metric\",\"language\":\"kuery\"},\"legend_position\":\"right\",\"background_color\":null}}", + "visState": "{\"title\":\"Zeek and Suricata Capture Measurements \",\"type\":\"metrics\",\"aggs\":[],\"params\":{\"id\":\"61ca57f0-469d-11e7-af02-69e470af7417\",\"type\":\"timeseries\",\"series\":[{\"id\":\"32d1fca0-d7e1-11ee-ad81-217e54128a4b\",\"color\":\"rgba(33,150,243,1)\",\"split_mode\":\"everything\",\"metrics\":[{\"id\":\"32d1fca1-d7e1-11ee-ad81-217e54128a4b\",\"type\":\"sum\",\"field\":\"zeek.stats.pkts_link\"}],\"separate_axis\":0,\"axis_position\":\"right\",\"formatter\":\"number\",\"chart_type\":\"line\",\"line_width\":\"0\",\"point_size\":\"0\",\"fill\":0.5,\"stacked\":\"none\",\"label\":\"Zeek: packets seen\",\"type\":\"timeseries\",\"terms_field\":\"host.name\",\"terms_size\":\"25\",\"terms_order_by\":\"32d1fca1-d7e1-11ee-ad81-217e54128a4b\",\"split_color_mode\":\"opensearchDashboards\"},{\"id\":\"02bbf6a0-d7fb-11ee-a5f1-9ff9da698a18\",\"color\":\"rgba(84,179,153,1)\",\"split_mode\":\"everything\",\"metrics\":[{\"unit\":\"\",\"id\":\"02bbf6a1-d7fb-11ee-a5f1-9ff9da698a18\",\"type\":\"positive_rate\",\"field\":\"suricata.stats.capture.kernel_packets\"}],\"separate_axis\":0,\"axis_position\":\"right\",\"formatter\":\"number\",\"chart_type\":\"line\",\"line_width\":\"0\",\"point_size\":\"0\",\"fill\":0.5,\"stacked\":\"none\",\"label\":\"Suricata: packets seen\",\"type\":\"timeseries\"},{\"id\":\"e4143600-d7e0-11ee-ad81-217e54128a4b\",\"color\":\"rgba(229,115,115,1)\",\"split_mode\":\"everything\",\"metrics\":[{\"id\":\"e4143601-d7e0-11ee-ad81-217e54128a4b\",\"type\":\"sum\",\"field\":\"zeek.stats.pkts_dropped\"},{\"id\":\"f6df2790-d7e0-11ee-ad81-217e54128a4b\",\"type\":\"math\",\"variables\":[{\"id\":\"f8ee0a60-d7e0-11ee-ad81-217e54128a4b\",\"name\":\"packets\",\"field\":\"e4143601-d7e0-11ee-ad81-217e54128a4b\"}],\"script\":\"params.packets*-1\"}],\"separate_axis\":0,\"axis_position\":\"right\",\"formatter\":\"number\",\"chart_type\":\"line\",\"line_width\":\"0\",\"point_size\":\"0\",\"fill\":0.5,\"stacked\":\"none\",\"label\":\"Zeek: packets dropped\",\"type\":\"timeseries\",\"terms_field\":\"host.name\",\"terms_size\":\"25\",\"terms_order_by\":\"e4143601-d7e0-11ee-ad81-217e54128a4b\",\"split_color_mode\":\"opensearchDashboards\"},{\"id\":\"20b9a420-d7df-11ee-ad81-217e54128a4b\",\"color\":\"rgba(211,96,134,1)\",\"split_mode\":\"everything\",\"metrics\":[{\"id\":\"20b9a421-d7df-11ee-ad81-217e54128a4b\",\"type\":\"sum\",\"field\":\"zeek.capture_loss.gaps\"},{\"id\":\"9a3afce0-d7df-11ee-ad81-217e54128a4b\",\"type\":\"math\",\"variables\":[{\"id\":\"9dece150-d7df-11ee-ad81-217e54128a4b\",\"name\":\"gaps\",\"field\":\"20b9a421-d7df-11ee-ad81-217e54128a4b\"}],\"script\":\"params.gaps*-1\"}],\"separate_axis\":0,\"axis_position\":\"right\",\"formatter\":\"number\",\"chart_type\":\"line\",\"line_width\":\"0\",\"point_size\":\"0\",\"fill\":0.5,\"stacked\":\"none\",\"label\":\"Zeek: ACKS missed\",\"type\":\"timeseries\",\"terms_field\":\"host.name\",\"terms_size\":\"25\",\"terms_order_by\":\"20b9a421-d7df-11ee-ad81-217e54128a4b\",\"split_color_mode\":\"opensearchDashboards\"},{\"id\":\"cad40600-d7fb-11ee-a5f1-9ff9da698a18\",\"color\":\"rgba(255,171,145,1)\",\"split_mode\":\"everything\",\"metrics\":[{\"unit\":\"\",\"id\":\"cad40601-d7fb-11ee-a5f1-9ff9da698a18\",\"type\":\"positive_rate\",\"field\":\"suricata.stats.pkts_dropped\"},{\"id\":\"f5352cd0-d7fb-11ee-a5f1-9ff9da698a18\",\"type\":\"math\",\"variables\":[{\"id\":\"f79def70-d7fb-11ee-a5f1-9ff9da698a18\",\"name\":\"packets\",\"field\":\"cad40601-d7fb-11ee-a5f1-9ff9da698a18\"}],\"script\":\"params.packets*-1\"}],\"separate_axis\":0,\"axis_position\":\"right\",\"formatter\":\"number\",\"chart_type\":\"line\",\"line_width\":\"0\",\"point_size\":\"0\",\"fill\":0.5,\"stacked\":\"none\",\"label\":\"Suricata: packets dropped\",\"type\":\"timeseries\"}],\"time_field\":\"@timestamp\",\"index_pattern\":\"malcolm_beats_*\",\"interval\":\"\",\"axis_position\":\"left\",\"axis_formatter\":\"number\",\"axis_scale\":\"normal\",\"show_legend\":1,\"show_grid\":1,\"tooltip_mode\":\"show_all\",\"default_index_pattern\":\"arkime_sessions3-*\",\"default_timefield\":\"firstPacket\",\"isModelInvalid\":false,\"filter\":{\"query\":\"(event.provider:zeek OR event.provider:suricata) AND event.kind:metric\",\"language\":\"kuery\"},\"legend_position\":\"right\",\"background_color\":null}}", "uiStateJSON": "{}", "description": "Positive values on the y-axis represent observed packets while negative values represent missing dropped packets and missing ACKs.\n\nThis data is logged by Zeek in stats.log (https://docs.zeek.org/en/master/scripts/policy/misc/stats.zeek.html#type-Stats::Info) and capture_loss.log (https://docs.zeek.org/en/master/scripts/policy/misc/capture-loss.zeek.html#type-CaptureLoss::Info), and by Suricata (https://docs.suricata.io/en/suricata-6.0.0/performance/statistics.html).", "version": 1, @@ -160,8 +160,8 @@ "namespaces": [ "default" ], - "updated_at": "2024-03-01T16:02:17.775Z", - "version": "Wzk2OSwxXQ==", + "updated_at": "2024-03-04T21:05:53.644Z", + "version": "Wzg4NywxXQ==", "attributes": { "title": "Zeek Stats - Packets and Bytes", "visState": "{\"title\":\"Zeek Stats - Packets and Bytes\",\"type\":\"metrics\",\"aggs\":[],\"params\":{\"id\":\"61ca57f0-469d-11e7-af02-69e470af7417\",\"type\":\"metric\",\"series\":[{\"id\":\"61ca57f1-469d-11e7-af02-69e470af7417\",\"color\":\"#54B399\",\"split_mode\":\"everything\",\"split_color_mode\":\"opensearchDashboards\",\"metrics\":[{\"id\":\"61ca57f2-469d-11e7-af02-69e470af7417\",\"type\":\"sum\",\"field\":\"zeek.stats.pkts_link\"}],\"separate_axis\":0,\"axis_position\":\"right\",\"formatter\":\"number\",\"chart_type\":\"line\",\"line_width\":1,\"point_size\":1,\"fill\":0.5,\"stacked\":\"none\",\"label\":\"Total Packets Seen\"},{\"id\":\"bd4560e0-d7e4-11ee-ad81-217e54128a4b\",\"color\":\"#68BC00\",\"split_mode\":\"everything\",\"metrics\":[{\"id\":\"bd4560e1-d7e4-11ee-ad81-217e54128a4b\",\"type\":\"sum\",\"field\":\"zeek.stats.bytes_recv\"}],\"separate_axis\":0,\"axis_position\":\"right\",\"formatter\":\"bytes\",\"chart_type\":\"line\",\"line_width\":1,\"point_size\":1,\"fill\":0.5,\"stacked\":\"none\",\"label\":\"Total Bytes Seen\"}],\"time_field\":\"@timestamp\",\"index_pattern\":\"malcolm_beats_*\",\"interval\":\"\",\"axis_position\":\"left\",\"axis_formatter\":\"number\",\"axis_scale\":\"normal\",\"show_legend\":1,\"show_grid\":1,\"tooltip_mode\":\"show_all\",\"default_index_pattern\":\"arkime_sessions3-*\",\"default_timefield\":\"firstPacket\",\"isModelInvalid\":false,\"background_color_rules\":[{\"id\":\"28bcc800-d7e4-11ee-ad81-217e54128a4b\"}],\"filter\":{\"query\":\"event.provider:zeek AND event.dataset:stats\",\"language\":\"kuery\"},\"time_range_mode\":\"entire_time_range\"}}", @@ -183,8 +183,8 @@ "namespaces": [ "default" ], - "updated_at": "2024-03-01T16:12:29.036Z", - "version": "Wzk3NCwxXQ==", + "updated_at": "2024-03-04T21:05:53.644Z", + "version": "Wzg4OCwxXQ==", "attributes": { "title": "Zeek Stats - Capture Loss", "visState": "{\"title\":\"Zeek Stats - Capture Loss\",\"type\":\"metrics\",\"aggs\":[],\"params\":{\"id\":\"61ca57f0-469d-11e7-af02-69e470af7417\",\"type\":\"metric\",\"series\":[{\"id\":\"61ca57f1-469d-11e7-af02-69e470af7417\",\"color\":\"#54B399\",\"split_mode\":\"everything\",\"split_color_mode\":\"opensearchDashboards\",\"metrics\":[{\"id\":\"61ca57f2-469d-11e7-af02-69e470af7417\",\"type\":\"sum\",\"field\":\"zeek.stats.pkts_dropped\"}],\"separate_axis\":0,\"axis_position\":\"right\",\"formatter\":\"number\",\"chart_type\":\"line\",\"line_width\":1,\"point_size\":1,\"fill\":0.5,\"stacked\":\"none\",\"label\":\"Packets Dropped\"},{\"id\":\"bd4560e0-d7e4-11ee-ad81-217e54128a4b\",\"color\":\"#68BC00\",\"split_mode\":\"everything\",\"metrics\":[{\"id\":\"bd4560e1-d7e4-11ee-ad81-217e54128a4b\",\"type\":\"sum\",\"field\":\"zeek.capture_loss.gaps\"}],\"separate_axis\":0,\"axis_position\":\"right\",\"formatter\":\"number\",\"chart_type\":\"line\",\"line_width\":1,\"point_size\":1,\"fill\":0.5,\"stacked\":\"none\",\"label\":\"ACKs Missed\"}],\"time_field\":\"@timestamp\",\"index_pattern\":\"malcolm_beats_*\",\"interval\":\"\",\"axis_position\":\"left\",\"axis_formatter\":\"number\",\"axis_scale\":\"normal\",\"show_legend\":1,\"show_grid\":1,\"tooltip_mode\":\"show_all\",\"default_index_pattern\":\"arkime_sessions3-*\",\"default_timefield\":\"firstPacket\",\"isModelInvalid\":false,\"background_color_rules\":[{\"id\":\"28bcc800-d7e4-11ee-ad81-217e54128a4b\"}],\"filter\":{\"query\":\"event.provider:zeek AND event.dataset:(stats OR capture_loss)\",\"language\":\"kuery\"},\"time_range_mode\":\"entire_time_range\"}}", @@ -206,8 +206,8 @@ "namespaces": [ "default" ], - "updated_at": "2024-03-01T15:01:44.258Z", - "version": "Wzg4MywxXQ==", + "updated_at": "2024-03-04T21:05:53.644Z", + "version": "Wzg4OSwxXQ==", "attributes": { "title": "Packet Capture - Zeek capture_loss.log", "description": "", @@ -243,8 +243,8 @@ "namespaces": [ "default" ], - "updated_at": "2024-03-01T18:56:30.009Z", - "version": "WzEwMDksMV0=", + "updated_at": "2024-03-04T21:05:53.644Z", + "version": "Wzg5MCwxXQ==", "attributes": { "title": "Suricata Stats - Packets and Bytes", "visState": "{\"title\":\"Suricata Stats - Packets and Bytes\",\"type\":\"metrics\",\"aggs\":[],\"params\":{\"id\":\"61ca57f0-469d-11e7-af02-69e470af7417\",\"type\":\"metric\",\"series\":[{\"id\":\"61ca57f1-469d-11e7-af02-69e470af7417\",\"color\":\"#54B399\",\"split_mode\":\"everything\",\"split_color_mode\":\"opensearchDashboards\",\"metrics\":[{\"size\":1,\"agg_with\":\"max\",\"order\":\"desc\",\"unit\":\"\",\"id\":\"61ca57f2-469d-11e7-af02-69e470af7417\",\"type\":\"top_hit\",\"field\":\"suricata.stats.capture.kernel_packets\",\"order_by\":\"@timestamp\"}],\"separate_axis\":0,\"axis_position\":\"right\",\"formatter\":\"number\",\"chart_type\":\"line\",\"line_width\":1,\"point_size\":1,\"fill\":0.5,\"stacked\":\"none\",\"label\":\"Total Packets Seen\"},{\"id\":\"bd4560e0-d7e4-11ee-ad81-217e54128a4b\",\"color\":\"#68BC00\",\"split_mode\":\"everything\",\"metrics\":[{\"size\":1,\"agg_with\":\"max\",\"order\":\"desc\",\"id\":\"bd4560e1-d7e4-11ee-ad81-217e54128a4b\",\"type\":\"top_hit\",\"field\":\"suricata.stats.decoder.bytes\",\"order_by\":\"@timestamp\"}],\"separate_axis\":0,\"axis_position\":\"right\",\"formatter\":\"bytes\",\"chart_type\":\"line\",\"line_width\":1,\"point_size\":1,\"fill\":0.5,\"stacked\":\"none\",\"label\":\"Total Bytes Seen\"}],\"time_field\":\"@timestamp\",\"index_pattern\":\"malcolm_beats_*\",\"interval\":\"\",\"axis_position\":\"left\",\"axis_formatter\":\"number\",\"axis_scale\":\"normal\",\"show_legend\":1,\"show_grid\":1,\"tooltip_mode\":\"show_all\",\"default_index_pattern\":\"arkime_sessions3-*\",\"default_timefield\":\"firstPacket\",\"isModelInvalid\":false,\"background_color_rules\":[{\"id\":\"28bcc800-d7e4-11ee-ad81-217e54128a4b\"}],\"filter\":{\"query\":\"event.provider:suricata AND event.kind:metric\",\"language\":\"kuery\"},\"time_range_mode\":\"entire_time_range\"}}", @@ -266,11 +266,11 @@ "namespaces": [ "default" ], - "updated_at": "2024-03-01T19:02:50.252Z", - "version": "WzEwNzksMV0=", + "updated_at": "2024-03-04T21:07:12.009Z", + "version": "Wzk5NiwxXQ==", "attributes": { "title": "Suricata Stats - Capture Loss", - "visState": "{\"title\":\"Suricata Stats - Capture Loss\",\"type\":\"metrics\",\"aggs\":[],\"params\":{\"id\":\"61ca57f0-469d-11e7-af02-69e470af7417\",\"type\":\"metric\",\"series\":[{\"id\":\"61ca57f1-469d-11e7-af02-69e470af7417\",\"color\":\"#54B399\",\"split_mode\":\"everything\",\"split_color_mode\":\"opensearchDashboards\",\"metrics\":[{\"size\":1,\"agg_with\":\"max\",\"order\":\"desc\",\"id\":\"61ca57f2-469d-11e7-af02-69e470af7417\",\"type\":\"top_hit\",\"field\":\"suricata.stats.capture.kernel_drops\",\"order_by\":\"@timestamp\"},{\"size\":1,\"agg_with\":\"max\",\"order\":\"desc\",\"id\":\"ffb34320-d7fd-11ee-a5f1-9ff9da698a18\",\"type\":\"top_hit\",\"field\":\"suricata.stats.tcp.segment_memcap_drop\",\"order_by\":\"@timestamp\"},{\"size\":1,\"agg_with\":\"max\",\"order\":\"desc\",\"id\":\"0e962510-d7fe-11ee-a5f1-9ff9da698a18\",\"type\":\"top_hit\",\"field\":\"suricata.stats.tcp.ssn_memcap_drop\",\"order_by\":\"@timestamp\"},{\"id\":\"1d60c410-d7fe-11ee-a5f1-9ff9da698a18\",\"type\":\"math\",\"variables\":[{\"id\":\"21851960-d7fe-11ee-a5f1-9ff9da698a18\",\"name\":\"kernel\",\"field\":\"61ca57f2-469d-11e7-af02-69e470af7417\"},{\"id\":\"25a01fe0-d7fe-11ee-a5f1-9ff9da698a18\",\"name\":\"segment\",\"field\":\"ffb34320-d7fd-11ee-a5f1-9ff9da698a18\"},{\"id\":\"2b81b590-d7fe-11ee-a5f1-9ff9da698a18\",\"name\":\"session\",\"field\":\"0e962510-d7fe-11ee-a5f1-9ff9da698a18\"}],\"script\":\"params.kernel+params.segment+params.session\"}],\"separate_axis\":0,\"axis_position\":\"right\",\"formatter\":\"number\",\"chart_type\":\"line\",\"line_width\":1,\"point_size\":1,\"fill\":0.5,\"stacked\":\"none\",\"label\":\"Packets Dropped\"}],\"time_field\":\"@timestamp\",\"index_pattern\":\"malcolm_beats_*\",\"interval\":\"\",\"axis_position\":\"left\",\"axis_formatter\":\"number\",\"axis_scale\":\"normal\",\"show_legend\":1,\"show_grid\":1,\"tooltip_mode\":\"show_all\",\"default_index_pattern\":\"arkime_sessions3-*\",\"default_timefield\":\"firstPacket\",\"isModelInvalid\":false,\"background_color_rules\":[{\"id\":\"28bcc800-d7e4-11ee-ad81-217e54128a4b\"}],\"filter\":{\"query\":\"event.provider:suricata AND event.kind:metric\",\"language\":\"kuery\"},\"time_range_mode\":\"entire_time_range\"}}", + "visState": "{\"title\":\"Suricata Stats - Capture Loss\",\"type\":\"metrics\",\"aggs\":[],\"params\":{\"id\":\"61ca57f0-469d-11e7-af02-69e470af7417\",\"type\":\"metric\",\"series\":[{\"id\":\"61ca57f1-469d-11e7-af02-69e470af7417\",\"color\":\"#54B399\",\"split_mode\":\"everything\",\"split_color_mode\":\"opensearchDashboards\",\"metrics\":[{\"size\":1,\"agg_with\":\"max\",\"order\":\"desc\",\"id\":\"61ca57f2-469d-11e7-af02-69e470af7417\",\"type\":\"top_hit\",\"field\":\"suricata.stats.pkts_dropped\",\"order_by\":\"@timestamp\"}],\"separate_axis\":0,\"axis_position\":\"right\",\"formatter\":\"number\",\"chart_type\":\"line\",\"line_width\":1,\"point_size\":1,\"fill\":0.5,\"stacked\":\"none\",\"label\":\"Packets Dropped\"}],\"time_field\":\"@timestamp\",\"index_pattern\":\"malcolm_beats_*\",\"interval\":\"\",\"axis_position\":\"left\",\"axis_formatter\":\"number\",\"axis_scale\":\"normal\",\"show_legend\":1,\"show_grid\":1,\"tooltip_mode\":\"show_all\",\"default_index_pattern\":\"arkime_sessions3-*\",\"default_timefield\":\"firstPacket\",\"isModelInvalid\":false,\"background_color_rules\":[{\"id\":\"28bcc800-d7e4-11ee-ad81-217e54128a4b\"}],\"filter\":{\"query\":\"event.provider:suricata AND event.kind:metric\",\"language\":\"kuery\"},\"time_range_mode\":\"entire_time_range\"}}", "uiStateJSON": "{}", "description": "", "version": 1, @@ -289,8 +289,8 @@ "namespaces": [ "default" ], - "updated_at": "2024-03-01T15:01:49.331Z", - "version": "WzkxNywxXQ==", + "updated_at": "2024-03-04T21:05:59.817Z", + "version": "Wzk0MCwxXQ==", "attributes": { "title": "Network Traffic (Packets)", "visState": "{\"title\":\"Network Traffic (Packets)\",\"type\":\"metrics\",\"aggs\":[],\"params\":{\"axis_formatter\":\"number\",\"axis_position\":\"left\",\"axis_scale\":\"normal\",\"default_index_pattern\":\"arkime_sessions3-*\",\"default_timefield\":\"firstPacket\",\"filter\":{\"language\":\"lucene\",\"query\":\"\"},\"id\":\"da1046f0-faa0-11e6-86b1-cd7735ff7e23\",\"index_pattern\":\"malcolm_beats_*\",\"interval\":\"auto\",\"isModelInvalid\":false,\"series\":[{\"axis_position\":\"right\",\"chart_type\":\"line\",\"color\":\"rgba(0,133,255,1)\",\"fill\":\"1\",\"formatter\":\"'0a'\",\"id\":\"49931900-ebf3-11ec-a401-f5db2d59e6af\",\"label\":\"Inbound\",\"line_width\":1,\"metrics\":[{\"unit\":\"1s\",\"id\":\"49931901-ebf3-11ec-a401-f5db2d59e6af\",\"type\":\"positive_rate\",\"field\":\"miscbeat.network.packets.rx\"}],\"point_size\":1,\"separate_axis\":0,\"split_mode\":\"terms\",\"stacked\":\"none\",\"type\":\"timeseries\",\"terms_field\":\"miscbeat.network.interface\",\"terms_size\":\"3\",\"terms_order_by\":\"_count\",\"value_template\":\"{{value}}/s\",\"split_color_mode\":\"gradient\"},{\"id\":\"75fba890-ebf3-11ec-a401-f5db2d59e6af\",\"color\":\"rgba(13,212,26,1)\",\"split_mode\":\"terms\",\"metrics\":[{\"unit\":\"1s\",\"id\":\"75fba891-ebf3-11ec-a401-f5db2d59e6af\",\"type\":\"positive_rate\",\"field\":\"miscbeat.network.packets.tx\"},{\"id\":\"96daba60-ebf3-11ec-a401-f5db2d59e6af\",\"type\":\"math\",\"variables\":[{\"id\":\"98e138c0-ebf3-11ec-a401-f5db2d59e6af\",\"name\":\"rate\",\"field\":\"75fba891-ebf3-11ec-a401-f5db2d59e6af\"}],\"script\":\"params.rate*-1\"}],\"separate_axis\":0,\"axis_position\":\"right\",\"formatter\":\"'0a'\",\"chart_type\":\"line\",\"line_width\":1,\"point_size\":1,\"fill\":\"1\",\"stacked\":\"none\",\"label\":\"Outbound\",\"type\":\"timeseries\",\"terms_size\":\"3\",\"terms_field\":\"miscbeat.network.interface\",\"terms_order_by\":\"_count\",\"split_color_mode\":\"gradient\",\"value_template\":\"{{value}}/s\"}],\"show_grid\":1,\"show_legend\":1,\"time_field\":\"@timestamp\",\"tooltip_mode\":\"show_all\",\"type\":\"timeseries\"}}", @@ -312,8 +312,8 @@ "namespaces": [ "default" ], - "updated_at": "2024-03-01T15:01:44.258Z", - "version": "Wzg4NCwxXQ==", + "updated_at": "2024-03-04T21:05:53.644Z", + "version": "Wzg5MywxXQ==", "attributes": { "title": "Packet Capture - Zeek stats.log", "description": "", @@ -354,8 +354,8 @@ "namespaces": [ "default" ], - "updated_at": "2024-03-01T18:37:00.084Z", - "version": "Wzk5NCwxXQ==", + "updated_at": "2024-03-04T21:05:53.644Z", + "version": "Wzg5NCwxXQ==", "attributes": { "title": "Packet Capture - Suricata Stats", "description": "", @@ -363,7 +363,7 @@ "columns": [ "host.name", "suricata.stats.capture.kernel_packets", - "suricata.stats.capture.kernel_drops", + "suricata.stats.pkts_dropped", "suricata.stats.capture.errors", "suricata.stats.decoder.bytes", "suricata.stats.decoder.ethernet", @@ -395,8 +395,8 @@ "namespaces": [ "default" ], - "updated_at": "2024-03-01T19:07:58.499Z", - "version": "WzEwOTEsMV0=", + "updated_at": "2024-03-04T21:05:53.644Z", + "version": "Wzg5NSwxXQ==", "attributes": { "title": "Zeek Analyzer Messages", "visState": "{\"title\":\"Zeek Analyzer Messages\",\"type\":\"table\",\"aggs\":[{\"id\":\"1\",\"enabled\":true,\"type\":\"count\",\"params\":{},\"schema\":\"metric\"},{\"id\":\"4\",\"enabled\":true,\"type\":\"terms\",\"params\":{\"field\":\"zeek.analyzer.cause\",\"orderBy\":\"1\",\"order\":\"desc\",\"size\":50,\"otherBucket\":true,\"otherBucketLabel\":\"Other\",\"missingBucket\":true,\"missingBucketLabel\":\"-\",\"customLabel\":\"Cause\"},\"schema\":\"bucket\"},{\"id\":\"2\",\"enabled\":true,\"type\":\"terms\",\"params\":{\"field\":\"zeek.analyzer.analyzer_kind\",\"orderBy\":\"1\",\"order\":\"desc\",\"size\":25,\"otherBucket\":true,\"otherBucketLabel\":\"Other\",\"missingBucket\":true,\"missingBucketLabel\":\"-\",\"customLabel\":\"Class\"},\"schema\":\"bucket\"},{\"id\":\"3\",\"enabled\":true,\"type\":\"terms\",\"params\":{\"field\":\"zeek.analyzer.analyzer_name\",\"orderBy\":\"1\",\"order\":\"desc\",\"size\":100,\"otherBucket\":true,\"otherBucketLabel\":\"Other\",\"missingBucket\":true,\"missingBucketLabel\":\"-\",\"customLabel\":\"Analyzer\"},\"schema\":\"bucket\"}],\"params\":{\"perPage\":10,\"showPartialRows\":false,\"showMetricsAtAllLevels\":false,\"showTotal\":false,\"totalFunc\":\"sum\",\"percentageCol\":\"\"}}", @@ -425,8 +425,8 @@ "namespaces": [ "default" ], - "updated_at": "2024-03-01T15:01:44.258Z", - "version": "Wzg4MiwxXQ==", + "updated_at": "2024-03-04T21:05:53.644Z", + "version": "Wzg5NiwxXQ==", "attributes": { "title": "Packet Capture - Zeek analyzer.log", "description": "", @@ -464,8 +464,8 @@ "namespaces": [ "default" ], - "updated_at": "2024-03-01T15:52:59.726Z", - "version": "Wzk2OCwxXQ==", + "updated_at": "2024-03-04T21:05:53.644Z", + "version": "Wzg5NywxXQ==", "attributes": { "title": "Zeek - Reporter Categories", "visState": "{\"title\":\"Zeek - Reporter Categories\",\"type\":\"pie\",\"aggs\":[{\"id\":\"1\",\"enabled\":true,\"type\":\"count\",\"params\":{},\"schema\":\"metric\"},{\"id\":\"2\",\"enabled\":true,\"type\":\"terms\",\"params\":{\"field\":\"zeek.reporter.level.keyword\",\"orderBy\":\"1\",\"order\":\"desc\",\"size\":10,\"otherBucket\":true,\"otherBucketLabel\":\"Other\",\"missingBucket\":false,\"missingBucketLabel\":\"Missing\"},\"schema\":\"segment\"}],\"params\":{\"type\":\"pie\",\"addTooltip\":true,\"addLegend\":true,\"legendPosition\":\"right\",\"isDonut\":false,\"labels\":{\"show\":true,\"values\":true,\"last_level\":true,\"truncate\":100}}}", @@ -494,8 +494,8 @@ "namespaces": [ "default" ], - "updated_at": "2024-03-01T15:01:44.258Z", - "version": "Wzg4NSwxXQ==", + "updated_at": "2024-03-04T21:05:53.644Z", + "version": "Wzg5OCwxXQ==", "attributes": { "title": "Packet Capture - Zeek reporter.log", "description": "", @@ -529,8 +529,8 @@ "namespaces": [ "default" ], - "updated_at": "2024-03-01T15:01:49.331Z", - "version": "WzkxOCwxXQ==", + "updated_at": "2024-03-04T21:05:59.817Z", + "version": "Wzk0MSwxXQ==", "attributes": { "title": "Network Traffic (Bytes)", "visState": "{\"title\":\"Network Traffic (Bytes)\",\"type\":\"metrics\",\"aggs\":[],\"params\":{\"axis_formatter\":\"number\",\"axis_position\":\"left\",\"axis_scale\":\"normal\",\"default_index_pattern\":\"arkime_sessions3-*\",\"default_timefield\":\"firstPacket\",\"filter\":{\"language\":\"lucene\",\"query\":\"\"},\"id\":\"da1046f0-faa0-11e6-86b1-cd7735ff7e23\",\"index_pattern\":\"malcolm_beats_*\",\"interval\":\"auto\",\"isModelInvalid\":false,\"series\":[{\"axis_position\":\"right\",\"chart_type\":\"line\",\"color\":\"rgba(0,133,255,1)\",\"fill\":\"1\",\"formatter\":\"bytes\",\"id\":\"6d8b8ab0-ebf1-11ec-a401-f5db2d59e6af\",\"line_width\":1,\"metrics\":[{\"unit\":\"1s\",\"id\":\"6d8b8ab1-ebf1-11ec-a401-f5db2d59e6af\",\"type\":\"positive_rate\",\"field\":\"miscbeat.network.bytes.rx\"}],\"point_size\":1,\"separate_axis\":0,\"split_mode\":\"terms\",\"stacked\":\"none\",\"label\":\"Inbound\",\"type\":\"timeseries\",\"terms_field\":\"miscbeat.network.interface\",\"terms_size\":\"3\",\"terms_order_by\":\"_key\",\"value_template\":\"{{value}}/s\",\"split_color_mode\":\"gradient\"},{\"id\":\"b5977de0-ebf2-11ec-a401-f5db2d59e6af\",\"color\":\"rgba(13,212,26,1)\",\"split_mode\":\"terms\",\"metrics\":[{\"unit\":\"1s\",\"id\":\"b5977de1-ebf2-11ec-a401-f5db2d59e6af\",\"type\":\"positive_rate\",\"field\":\"miscbeat.network.bytes.tx\"},{\"id\":\"cdfb1540-ebf2-11ec-a401-f5db2d59e6af\",\"type\":\"math\",\"variables\":[{\"id\":\"d1b9caf0-ebf2-11ec-a401-f5db2d59e6af\",\"name\":\"rate\",\"field\":\"b5977de1-ebf2-11ec-a401-f5db2d59e6af\"}],\"script\":\"params.rate*-1\"}],\"separate_axis\":0,\"axis_position\":\"right\",\"formatter\":\"bytes\",\"chart_type\":\"line\",\"line_width\":1,\"point_size\":1,\"fill\":\"1\",\"stacked\":\"none\",\"label\":\"Outbound\",\"split_color_mode\":\"gradient\",\"type\":\"timeseries\",\"terms_size\":\"3\",\"terms_order_by\":\"_key\",\"terms_field\":\"miscbeat.network.interface\",\"value_template\":\"{{value}}/s\"}],\"show_grid\":1,\"show_legend\":1,\"time_field\":\"@timestamp\",\"tooltip_mode\":\"show_all\",\"type\":\"timeseries\"}}", diff --git a/dashboards/templates/composable/component/suricata_stats.json b/dashboards/templates/composable/component/suricata_stats.json index 03fb77b2c..3aeb82a91 100644 --- a/dashboards/templates/composable/component/suricata_stats.json +++ b/dashboards/templates/composable/component/suricata_stats.json @@ -169,6 +169,7 @@ "ftp.memuse": { "type": "long" }, "http.memcap": { "type": "long" }, "http.memuse": { "type": "long" }, + "pkts_dropped": { "type": "long" }, "tcp.insert_data_normal_fail": { "type": "long" }, "tcp.insert_data_overlap_fail": { "type": "long" }, "tcp.insert_list_fail": { "type": "long" }, diff --git a/hedgehog-raspi/sensor_install.sh b/hedgehog-raspi/sensor_install.sh index 95c04454e..f5bed9718 100644 --- a/hedgehog-raspi/sensor_install.sh +++ b/hedgehog-raspi/sensor_install.sh @@ -270,8 +270,12 @@ clean_up() { /opt/hedgehog_install_artifacts \ /opt/hooks \ /opt/patches \ - /root/.bash_history \ - /root/.wget-hsts \ + /root/.bash_history \ + /root/.wget-hsts \ + /root/.cache \ + /root/.local/share/gem \ + /root/.npm \ + "${DEBS_DIR}" \ /tmp/* find /var/log/ -type f -print0 2>/dev/null | \ xargs -0 -r -I XXX bash -c "file 'XXX' | grep -q text && > 'XXX'" diff --git a/logstash/pipelines/beats/11_beats_logs.conf b/logstash/pipelines/beats/11_beats_logs.conf index 979529161..2d9febd02 100644 --- a/logstash/pipelines/beats/11_beats_logs.conf +++ b/logstash/pipelines/beats/11_beats_logs.conf @@ -837,6 +837,17 @@ filter { # Suricata statistics # https://docs.suricata.io/en/suricata-6.0.2/configuration/suricata-yaml.html#stats + # sum packet drops into suricata.stats.pkts_dropped + ruby { + id => "ruby_miscbeat_suricata_stats_pkts_dropped" + code => " + pkts_dropped = event.get('[suricata][stats][capture][kernel_drops]').to_i + + event.get('[suricata][stats][tcp][segment_memcap_drop]').to_i + + event.get('[suricata][stats][tcp][ssn_memcap_drop]').to_i + event.set('[suricata][stats][pkts_dropped]', pkts_dropped) unless (pkts_dropped == 0) + " + } + if ([suricata][stats]) { # remove zero values from suricata stats ruby {