detect: move non-pf rules into special prefilter engines

Instead of having a per detection engine list of rule that couldn't be prefiltered, put those into special "prefilter" engines. For packet and frame rules this doesn't change much, it just removes some hard coded logic from the detect engine. For the packet non-prefilter rules in the "non-prefilter" special prefilter engine, add additional filtering for the packet variant. It can prefilter on alproto, dsize and dest port. The frame non-prefilter rules are added to a single engine, that per rule checks the alproto and the type. For app-layer, there is an engine per progress value, per app-layer protocol and per direction. This hooks app-layer non-prefilter rules into the app inspect logic at the correct "progress" hook. e.g. a rule like dns.query; bsize:1; Negated MPM rules will also fall into this category: dns.query; content:!"abc"; Are part of a special "generic list" app engine for dns, at the same progress hook as `dns.query`. This all results in a lot fewer checks: previous: -------------------------------------------------------------------------- Date: 1/29/2025 -- 10:22:25. Sorted by: number of checks. -------------------------------------------------------------------------- Num Rule Gid Rev Ticks % Checks Matches Max Ticks Avg Ticks Avg Match Avg No Match -------- ------------ -------- -------- ------------ ------ -------- -------- ----------- ----------- ----------- -------------- 1 20 1 0 181919672 11.85 588808 221 60454 308.96 2691.46 308.07 2 50 1 0 223455914 14.56 453104 418 61634 493.17 3902.59 490.02 3 60 1 0 185990683 12.12 453104 418 60950 410.48 1795.40 409.20 4 51 1 0 192436011 12.54 427028 6084 61223 450.64 2749.12 417.42 5 61 1 0 180401533 11.75 427028 6084 61093 422.46 2177.04 397.10 6 70 1 0 153899099 10.03 369836 0 61282 416.13 0.00 416.13 7 71 1 0 123389405 8.04 369836 12833 44921 333.63 2430.23 258.27 8 41 1 0 63889876 4.16 155824 12568 39138 410.01 1981.97 272.10 9 40 1 0 64149724 4.18 155818 210 39792 411.70 4349.57 406.38 10 10 1 0 70848850 4.62 65558 0 39544 1080.70 0.00 1080.70 11 11 1 0 94743878 6.17 65558 32214 60547 1445.19 2616.14 313.92 this commit: -------------------------------------------------------------------------- Date: 1/29/2025 -- 10:15:46. Sorted by: number of checks. -------------------------------------------------------------------------- Num Rule Gid Rev Ticks % Checks Matches Max Ticks Avg Ticks Avg Match Avg No Match -------- ------------ -------- -------- ------------ ------ -------- -------- ----------- ----------- ----------- -------------- 1 50 1 0 138776766 19.23 95920 418 167584 1446.80 3953.11 1435.83 2 60 1 0 97988084 13.58 95920 418 182817 1021.56 1953.63 1017.48 3 51 1 0 105318318 14.60 69838 6084 65649 1508.04 2873.38 1377.74 4 61 1 0 89571260 12.41 69838 6084 164632 1282.56 2208.41 1194.20 5 11 1 0 91132809 12.63 32779 32214 373569 2780.22 2785.58 2474.45 6 10 1 0 66095303 9.16 32779 0 56704 2016.39 0.00 2016.39 7 70 1 0 48107573 6.67 12928 0 42832 3721.19 0.00 3721.19 8 71 1 0 32308792 4.48 12928 12833 39565 2499.13 2510.05 1025.09 9 41 1 0 25546837 3.54 12886 12470 41479 1982.53 1980.84 2033.05 10 40 1 0 26069992 3.61 12886 210 38495 2023.13 4330.05 1984.91 11 20 1 0 639025 0.09 221 221 14750 2891.52 2891.52 0.00
OISF · Jan 31, 2025 · 02c82c2 · 02c82c2
1 parent 0c8d131
commit 02c82c2
Show file tree

Hide file tree

Showing 13 changed files with 489 additions and 316 deletions.
diff --git a/src/app-layer-frames.h b/src/app-layer-frames.h
@@ -26,6 +26,8 @@
 
 #include "rust.h"
 
+/** special value for matching any type */
+#define FRAME_ANY_TYPE 62
 /** max 63 to fit the 64 bit per protocol space */
 #define FRAME_STREAM_TYPE 63
 

diff --git a/src/decode.h b/src/decode.h
@@ -94,6 +94,9 @@ enum PktSrcEnum {
 
 #include "util-validate.h"
 
+/* for now a uint8_t is enough -- here in decode as it's part of the packet */
+#define SignatureMask uint8_t
+
 /* forward declarations */
 struct DetectionEngineThreadCtx_;
 typedef struct AppLayerThreadCtx_ AppLayerThreadCtx;
@@ -508,6 +511,7 @@ typedef struct Packet_
     /* coccinelle: Packet:flowflags:FLOW_PKT_ */
 
     uint8_t app_update_direction; // enum StreamUpdateDir
+    SignatureMask sig_mask;
 
     /* Pkt Flags */
     uint32_t flags;

diff --git a/src/detect-engine-build.c b/src/detect-engine-build.c
@@ -1990,8 +1990,6 @@ int SigPrepareStage4(DetectEngineCtx *de_ctx)
 
         PrefilterSetupRuleGroup(de_ctx, sgh);
 
-        SigGroupHeadBuildNonPrefilterArray(de_ctx, sgh);
-
         sgh->id = idx;
         cnt++;
     }
@@ -2002,7 +2000,7 @@ int SigPrepareStage4(DetectEngineCtx *de_ctx)
     if (de_ctx->decoder_event_sgh != NULL) {
         /* no need to set filestore count here as that would make a
          * signature not decode event only. */
-        SigGroupHeadBuildNonPrefilterArray(de_ctx, de_ctx->decoder_event_sgh);
+        PrefilterSetupRuleGroup(de_ctx, de_ctx->decoder_event_sgh);
     }
 
     int dump_grouping = 0;

diff --git a/src/detect-engine-frame.c b/src/detect-engine-frame.c
@@ -76,8 +76,9 @@ void DetectRunPrefilterFrame(DetectEngineThreadCtx *det_ctx, const SigGroupHead
     SCLogDebug("pcap_cnt %" PRIu64, p->pcap_cnt);
     PrefilterEngine *engine = sgh->frame_engines;
     do {
-        BUG_ON(engine->alproto == ALPROTO_UNKNOWN);
-        if (engine->alproto == alproto && engine->ctx.frame_type == frame->type) {
+        if ((engine->alproto == alproto || engine->alproto == ALPROTO_UNKNOWN) &&
+                (engine->ctx.frame_type == frame->type ||
+                        engine->ctx.frame_type == FRAME_ANY_TYPE)) {
             SCLogDebug("frame %p engine %p", frame, engine);
             PREFILTER_PROFILING_START(det_ctx);
             engine->cb.PrefilterFrame(det_ctx, engine->pectx, p, frames, frame);