SQUASH prefilter non-pf add frame type filter

Simplify tx: no more alproto check

src/detect-engine-prefilter.c

@@ -420,17 +420,20 @@ static int PrefilterSetupRuleGroupSortHelper(const void *a, const void *b)
 
 struct PrefilterNonPFDataSig {
     uint32_t sid : 30;
-    uint32_t type : 2; // 0 alproto 1 dport 2 dsize
+    uint32_t type : 2; /**< type for `value` field below: 0:alproto 1:dport 2:dsize */
     uint16_t value;
+    /* since we have 2 more bytes available due to padding, we can add some addition
+     * filters here. */
     union {
         struct {
             SignatureMask sig_mask;
         } pkt;
         struct {
-            uint8_t type; // TODO
+            /* filter for frame type */
+            uint8_t type;
         } frame;
         struct {
-            uint8_t foo; // TODO
+            uint8_t foo; // TODO unused
         } app;
     };
 };
@@ -443,15 +446,14 @@ struct PrefilterNonPFData {
 static void PrefilterTxNonPF(DetectEngineThreadCtx *det_ctx, const void *pectx, Packet *p, Flow *f,
         void *tx, const uint64_t tx_id, const AppLayerTxData *tx_data, const uint8_t flags)
 {
-    const uint16_t alproto = f->alproto;
     const struct PrefilterNonPFData *data = (const struct PrefilterNonPFData *)pectx;
     SCLogDebug("adding %u sids", data->size);
     for (uint32_t i = 0; i < data->size; i++) {
         const struct PrefilterNonPFDataSig *ds = &data->array[i];
-        if (ds->value == ALPROTO_UNKNOWN || AppProtoEquals(ds->value, alproto)) {
-            const uint32_t sid = ds->sid;
-            PrefilterAddSids(&det_ctx->pmq, &sid, 1);
-        }
+        DEBUG_VALIDATE_BUG_ON(ds->value == ALPROTO_UNKNOWN);
+        DEBUG_VALIDATE_BUG_ON(!AppProtoEquals(ds->value, f->alproto));
+        const uint32_t sid = ds->sid;
+        PrefilterAddSids(&det_ctx->pmq, &sid, 1);
     }
 }
 
@@ -480,7 +482,6 @@ void PrefilterPktNonPFStatsDump(void)
 #endif
 }
 
-/* TODO the old code checked alproto and mask */
 static void PrefilterPktNonPF(DetectEngineThreadCtx *det_ctx, Packet *p, const void *pectx)
 {
     const uint16_t alproto = p->flow ? p->flow->alproto : ALPROTO_UNKNOWN;
@@ -527,6 +528,10 @@ static void PrefilterPktNonPF(DetectEngineThreadCtx *det_ctx, Packet *p, const v
     }
 }
 
+/** \internal
+ *  \brief engine to select the non-prefilter rules for frames
+ *  Checks the alproto and type as well.
+ *  Direction needs no checking as the rule groups are per direction. */
 static void PrefilterFrameNonPF(DetectEngineThreadCtx *det_ctx, const void *pectx, Packet *p,
         const Frames *frames, const Frame *frame)
 {
@@ -536,7 +541,8 @@ static void PrefilterFrameNonPF(DetectEngineThreadCtx *det_ctx, const void *pect
     SCLogDebug("adding %u sids", data->size);
     for (uint32_t i = 0; i < data->size; i++) {
         const struct PrefilterNonPFDataSig *ds = &data->array[i];
-        if (ds->value == ALPROTO_UNKNOWN || AppProtoEquals(ds->value, alproto)) {
+        if (ds->frame.type == frame->type &&
+                (ds->value == ALPROTO_UNKNOWN || AppProtoEquals(ds->value, alproto))) {
             const uint32_t sid = ds->sid;
             PrefilterAddSids(&det_ctx->pmq, &sid, 1);
         }
@@ -652,6 +658,7 @@ int PrefilterSetupRuleGroup(DetectEngineCtx *de_ctx, SigGroupHead *sgh)
             continue;
         SCLogDebug("setting up sid %u for non-prefilter", s->id);
 
+        uint8_t frame_type = 0; /**< only a single type per rule */
         bool tx_non_pf = false;
         bool frame_non_pf = false;
         bool pkt_non_pf = false;
@@ -660,6 +667,18 @@ int PrefilterSetupRuleGroup(DetectEngineCtx *de_ctx, SigGroupHead *sgh)
             const DetectBufferType *buf = DetectEngineBufferTypeGetById(de_ctx, list_id);
             if (buf->frame) {
                 frame_non_pf = true;
+                for (DetectEngineFrameInspectionEngine *f = de_ctx->frame_inspect_engines;
+                        f != NULL; f = f->next) {
+                    if (!((((s->flags & SIG_FLAG_TOSERVER) != 0 && f->dir == 0) ||
+                                  ((s->flags & SIG_FLAG_TOCLIENT) != 0 && f->dir == 1)) &&
+                                list_id == (int)f->sm_list &&
+                                AppProtoEquals(s->alproto, f->alproto)))
+                        continue;
+
+                    SCLogDebug("frame '%s' type %u", buf->name, f->type);
+                    frame_type = f->type;
+                }
+
             } else if (buf->packet) {
                 pkt_non_pf = true;
             } else {
@@ -771,6 +790,7 @@ int PrefilterSetupRuleGroup(DetectEngineCtx *de_ctx, SigGroupHead *sgh)
         } else if (frame_non_pf) {
             frame_non_pf_array[frame_non_pf_array_size].sid = s->num;
             frame_non_pf_array[frame_non_pf_array_size].value = s->alproto;
+            frame_non_pf_array[frame_non_pf_array_size].frame.type = frame_type;
             frame_non_pf_array_size++;
         } else {
             BUG_ON(1);