From eb0000cd9fdeb3af2b7d76ec6b00c1666ca00496 Mon Sep 17 00:00:00 2001 From: Jason Ish Date: Thu, 10 Oct 2024 16:06:09 -0600 Subject: [PATCH 1/7] flow: add callbacks for flow init and flow updates Adds user registerable callbacks for flow initialization, flow update and flow finish. Some plugins, such as other DPI libraries like nDPI need a way to hook into these flow lifecycle events. Ticket: #7319 Ticket: #7320 --- src/Makefile.am | 2 + src/flow-callbacks.c | 129 +++++++++++++++++++++++++++++++++++++++++++ src/flow-callbacks.h | 121 ++++++++++++++++++++++++++++++++++++++++ src/flow-hash.c | 8 ++- src/flow-manager.c | 3 +- src/flow-util.c | 5 +- src/flow-util.h | 2 +- src/flow.c | 3 + 8 files changed, 267 insertions(+), 6 deletions(-) create mode 100644 src/flow-callbacks.c create mode 100644 src/flow-callbacks.h diff --git a/src/Makefile.am b/src/Makefile.am index 6970d709f35c..e6a7599037e0 100755 --- a/src/Makefile.am +++ b/src/Makefile.am @@ -328,6 +328,7 @@ noinst_HEADERS = \ feature.h \ flow-bit.h \ flow-bypass.h \ + flow-callbacks.h \ flow.h \ flow-hash.h \ flow-manager.h \ @@ -897,6 +898,7 @@ libsuricata_c_a_SOURCES = \ feature.c \ flow-bit.c \ flow-bypass.c \ + flow-callbacks.c \ flow.c \ flow-hash.c \ flow-manager.c \ diff --git a/src/flow-callbacks.c b/src/flow-callbacks.c new file mode 100644 index 000000000000..30e703c3efb3 --- /dev/null +++ b/src/flow-callbacks.c @@ -0,0 +1,129 @@ +/* Copyright (C) 2024 Open Information Security Foundation + * + * You can copy, redistribute or modify this Program under the terms of + * the GNU General Public License version 2 as published by the Free + * Software Foundation. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * version 2 along with this program; if not, write to the Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA + * 02110-1301, USA. + */ + +#include "flow-callbacks.h" + +typedef struct FlowInitCallback_ { + SCFlowInitCallbackFn Callback; + void *user; + struct FlowInitCallback_ *next; +} FlowInitCallback; + +static FlowInitCallback *init_callbacks = NULL; + +typedef struct FlowUpdateCallback_ { + SCFlowUpdateCallbackFn Callback; + void *user; + struct FlowUpdateCallback_ *next; +} FlowUpdateCallback; + +static FlowUpdateCallback *update_callbacks = NULL; + +typedef struct FlowFinishCallback_ { + SCFlowFinishCallbackFn Callback; + void *user; + struct FlowFinishCallback_ *next; +} FlowFinishCallback; + +static FlowFinishCallback *finish_callbacks = NULL; + +bool SCFlowRegisterInitCallback(SCFlowInitCallbackFn fn, void *user) +{ + FlowInitCallback *cb = SCCalloc(1, sizeof(*cb)); + if (cb == NULL) { + return false; + } + cb->Callback = fn; + cb->user = user; + if (init_callbacks == NULL) { + init_callbacks = cb; + } else { + FlowInitCallback *current = init_callbacks; + while (current->next != NULL) { + current = current->next; + } + current->next = cb; + } + return true; +} + +void SCFlowRunInitCallbacks(ThreadVars *tv, Flow *f, const Packet *p) +{ + FlowInitCallback *cb = init_callbacks; + while (cb != NULL) { + cb->Callback(tv, f, p, cb->user); + cb = cb->next; + } +} + +bool SCFlowRegisterUpdateCallback(SCFlowUpdateCallbackFn fn, void *user) +{ + FlowUpdateCallback *cb = SCCalloc(1, sizeof(*cb)); + if (cb == NULL) { + return false; + } + cb->Callback = fn; + cb->user = user; + if (update_callbacks == NULL) { + update_callbacks = cb; + } else { + FlowUpdateCallback *current = update_callbacks; + while (current->next != NULL) { + current = current->next; + } + current->next = cb; + } + return true; +} + +void SCFlowRunUpdateCallbacks(ThreadVars *tv, Flow *f, Packet *p) +{ + FlowUpdateCallback *cb = update_callbacks; + while (cb != NULL) { + cb->Callback(tv, f, p, cb->user); + cb = cb->next; + } +} + +bool SCFlowRegisterFinishCallback(SCFlowFinishCallbackFn fn, void *user) +{ + FlowFinishCallback *cb = SCCalloc(1, sizeof(*cb)); + if (cb == NULL) { + return false; + } + cb->Callback = fn; + cb->user = user; + if (finish_callbacks == NULL) { + finish_callbacks = cb; + } else { + FlowFinishCallback *current = finish_callbacks; + while (current->next != NULL) { + current = current->next; + } + current->next = cb; + } + return true; +} + +void SCFlowRunFinishCallbacks(ThreadVars *tv, Flow *f) +{ + FlowFinishCallback *cb = finish_callbacks; + while (cb != NULL) { + cb->Callback(tv, f, cb->user); + cb = cb->next; + } +} diff --git a/src/flow-callbacks.h b/src/flow-callbacks.h new file mode 100644 index 000000000000..4c694807753f --- /dev/null +++ b/src/flow-callbacks.h @@ -0,0 +1,121 @@ +/* Copyright (C) 2024 Open Information Security Foundation + * + * You can copy, redistribute or modify this Program under the terms of + * the GNU General Public License version 2 as published by the Free + * Software Foundation. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * version 2 along with this program; if not, write to the Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA + * 02110-1301, USA. + */ + +#ifndef SURICATA_FLOW_CALLBACKS_H +#define SURICATA_FLOW_CALLBACKS_H + +#include "suricata-common.h" +#include "flow.h" + +/** \brief Function type for flow initialization callbacks. + * + * Once registered with SCFlowRegisterInitCallback, this function will + * be called every time a flow is initialized, or in other words, + * every time Suricata picks up a flow. + * + * \param tv The ThreadVars data structure for the thread creating the + * flow. + * \param f The newly initialized flow. + * \param p The packet related to creating the new flow. + * \param user The user data provided during callback registration. + */ +typedef void (*SCFlowInitCallbackFn)(ThreadVars *tv, Flow *f, const Packet *p, void *user); + +/** \brief Register a flow init callback. + * + * Register a user provided function to be called every time a flow is + * initialized for use. + * + * \param fn Pointer to function to be called + * \param user Additional user data to be passed to callback + * + * \returns true if callback was registered, otherwise false if the + * callback could not be registered due to memory allocation error. + */ +bool SCFlowRegisterInitCallback(SCFlowInitCallbackFn fn, void *user); + +/** \internal + * + * Run all registered flow init callbacks. + */ +void SCFlowRunInitCallbacks(ThreadVars *tv, Flow *f, const Packet *p); + +/** \brief Function type for flow update callbacks. + * + * Once registered with SCFlowRegisterUpdateCallback, this function + * will be called every time a flow is updated by a packet (basically + * everytime a packet is seen on a flow). + * + * \param tv The ThreadVars data structure for the thread updating the + * flow. + * \param f The flow being updated. + * \param p The packet responsible for the flow update. + * \param user The user data provided during callback registration. + */ +typedef void (*SCFlowUpdateCallbackFn)(ThreadVars *tv, Flow *f, Packet *p, void *user); + +/** \brief Register a flow update callback. + * + * Register a user provided function to be called everytime a flow is + * updated. + * + * \param fn Pointer to function to be called + * \param user Additional user data to be passed to callback + * + * \returns true if callback was registered, otherwise false if the + * callback could not be registered due to memory allocation error. + */ +bool SCFlowRegisterUpdateCallback(SCFlowUpdateCallbackFn fn, void *user); + +/** \internal + * + * Run all registered flow update callbacks. + */ +void SCFlowRunUpdateCallbacks(ThreadVars *tv, Flow *f, Packet *p); + +/** \brief Function type for flow finish callbacks. + * + * Once registered with SCFlowRegisterFinshCallback, this function + * will be called when Suricata is done with a flow. + * + * \param tv The ThreadVars data structure for the thread finishing + * the flow. + * \param f The flow being finshed. + * \param user The user data provided during callback registration. + */ +typedef void (*SCFlowFinishCallbackFn)(ThreadVars *tv, Flow *f, void *user); + +/** \brief Register a flow init callback. + * + * Register a user provided function to be called every time a flow is + * finished. + * + * \param fn Pointer to function to be called + * \param user Additional user data to be passed to callback + * + * \returns true if callback was registered, otherwise false if the + * callback could not be registered due to memory allocation error. + */ +bool SCFlowRegisterFinishCallback(SCFlowFinishCallbackFn fn, void *user); + +/** \internal + * + * Run all registered flow init callbacks. + */ +void SCFlowRunFinishCallbacks(ThreadVars *tv, Flow *f); + +#endif /* SURICATA_FLOW_CALLBACKS_H */ diff --git a/src/flow-hash.c b/src/flow-hash.c index ddab01cd5b69..fcd957c72e27 100644 --- a/src/flow-hash.c +++ b/src/flow-hash.c @@ -38,6 +38,7 @@ #include "flow-storage.h" #include "flow-timeout.h" #include "flow-spare-pool.h" +#include "flow-callbacks.h" #include "app-layer-parser.h" #include "util-time.h" @@ -781,7 +782,7 @@ static Flow *TcpReuseReplace(ThreadVars *tv, FlowLookupStruct *fls, FlowBucket * fb->head = f; /* initialize and return */ - FlowInit(f, p); + FlowInit(tv, f, p); f->flow_hash = hash; f->fb = fb; FlowUpdateState(f, FLOW_STATE_NEW); @@ -886,7 +887,7 @@ Flow *FlowGetFlowFromHash(ThreadVars *tv, FlowLookupStruct *fls, Packet *p, Flow fb->head = f; /* got one, now lock, initialize and return */ - FlowInit(f, p); + FlowInit(tv, f, p); f->flow_hash = hash; f->fb = fb; FlowUpdateState(f, FLOW_STATE_NEW); @@ -951,7 +952,7 @@ Flow *FlowGetFlowFromHash(ThreadVars *tv, FlowLookupStruct *fls, Packet *p, Flow fb->head = f; /* initialize and return */ - FlowInit(f, p); + FlowInit(tv, f, p); f->flow_hash = hash; f->fb = fb; FlowUpdateState(f, FLOW_STATE_NEW); @@ -1242,6 +1243,7 @@ static Flow *FlowGetUsedFlow(ThreadVars *tv, DecodeThreadVars *dtv, const SCTime } #endif + SCFlowRunFinishCallbacks(tv, f); FlowClearMemory(f, f->protomap); /* leave locked */ diff --git a/src/flow-manager.c b/src/flow-manager.c index 05b791ee612e..9da986b22df6 100644 --- a/src/flow-manager.c +++ b/src/flow-manager.c @@ -39,6 +39,7 @@ #include "flow-manager.h" #include "flow-storage.h" #include "flow-spare-pool.h" +#include "flow-callbacks.h" #include "stream-tcp.h" #include "stream-tcp-cache.h" @@ -1059,7 +1060,7 @@ static void Recycler(ThreadVars *tv, FlowRecyclerThreadData *ftd, Flow *f) StatsDecr(tv, ftd->counter_tcp_active_sessions); } StatsDecr(tv, ftd->counter_flow_active); - + SCFlowRunFinishCallbacks(tv, f); FlowClearMemory(f, f->protomap); FLOWLOCK_UNLOCK(f); } diff --git a/src/flow-util.c b/src/flow-util.c index 7e11da41f527..31e22b9341ac 100644 --- a/src/flow-util.c +++ b/src/flow-util.c @@ -29,6 +29,7 @@ #include "flow.h" #include "flow-private.h" #include "flow-util.h" +#include "flow-callbacks.h" #include "flow-var.h" #include "app-layer.h" @@ -142,7 +143,7 @@ static inline void FlowSetICMPv6CounterPart(Flow *f) /* initialize the flow from the first packet * we see from it. */ -void FlowInit(Flow *f, const Packet *p) +void FlowInit(ThreadVars *tv, Flow *f, const Packet *p) { SCEnter(); SCLogDebug("flow %p", f); @@ -203,6 +204,8 @@ void FlowInit(Flow *f, const Packet *p) FlowSetStorageById(f, MacSetGetFlowStorageID(), ms); } + SCFlowRunInitCallbacks(tv, f, p); + SCReturn; } diff --git a/src/flow-util.h b/src/flow-util.h index 2d813bd9ee4d..368c955d876a 100644 --- a/src/flow-util.h +++ b/src/flow-util.h @@ -140,7 +140,7 @@ Flow *FlowAlloc(void); void FlowFree(Flow *); uint8_t FlowGetProtoMapping(uint8_t); -void FlowInit(Flow *, const Packet *); +void FlowInit(ThreadVars *, Flow *, const Packet *); uint8_t FlowGetReverseProtoMapping(uint8_t rproto); /* flow end counter logic */ diff --git a/src/flow.c b/src/flow.c index 7bfa80ea0a9b..aea79d23bf08 100644 --- a/src/flow.c +++ b/src/flow.c @@ -44,6 +44,7 @@ #include "flow-storage.h" #include "flow-bypass.h" #include "flow-spare-pool.h" +#include "flow-callbacks.h" #include "stream-tcp-private.h" @@ -503,6 +504,8 @@ void FlowHandlePacketUpdate(Flow *f, Packet *p, ThreadVars *tv, DecodeThreadVars SCLogDebug("setting FLOW_NOPAYLOAD_INSPECTION flag on flow %p", f); DecodeSetNoPayloadInspectionFlag(p); } + + SCFlowRunUpdateCallbacks(tv, f, p); } /** \brief Entry point for packet flow handling From b297f35d0796e4d47f9ddef3e5d5a74ea193a2fc Mon Sep 17 00:00:00 2001 From: Jason Ish Date: Fri, 11 Oct 2024 11:41:47 -0600 Subject: [PATCH 2/7] threads: add storage api, based on flow storage --- src/Makefile.am | 2 + src/thread-storage.c | 212 +++++++++++++++++++++++++++++++++++++++++++ src/thread-storage.h | 45 +++++++++ src/threads.c | 2 + src/threadvars.h | 2 + src/tm-threads.c | 5 +- src/util-storage.c | 2 + src/util-storage.h | 1 + 8 files changed, 270 insertions(+), 1 deletion(-) create mode 100644 src/thread-storage.c create mode 100644 src/thread-storage.h diff --git a/src/Makefile.am b/src/Makefile.am index e6a7599037e0..a2308c0de34f 100755 --- a/src/Makefile.am +++ b/src/Makefile.am @@ -449,6 +449,7 @@ noinst_HEADERS = \ suricata-common.h \ suricata.h \ suricata-plugin.h \ + thread-storage.h \ threads-debug.h \ threads.h \ threads-profile.h \ @@ -1012,6 +1013,7 @@ libsuricata_c_a_SOURCES = \ stream-tcp-sack.c \ stream-tcp-util.c \ suricata.c \ + thread-storage.c \ threads.c \ tm-modules.c \ tmqh-flow.c \ diff --git a/src/thread-storage.c b/src/thread-storage.c new file mode 100644 index 000000000000..977f4fde9752 --- /dev/null +++ b/src/thread-storage.c @@ -0,0 +1,212 @@ +/* Copyright (C) 2024 Open Information Security Foundation + * + * You can copy, redistribute or modify this Program under the terms of + * the GNU General Public License version 2 as published by the Free + * Software Foundation. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * version 2 along with this program; if not, write to the Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA + * 02110-1301, USA. + */ + +#include "suricata-common.h" +#include "thread-storage.h" +#include "util-storage.h" +#include "util-unittest.h" + +const StorageEnum storage_type = STORAGE_THREAD; + +unsigned int ThreadStorageSize(void) +{ + return StorageGetSize(storage_type); +} + +void *ThreadGetStorageById(const ThreadVars *tv, ThreadStorageId id) +{ + return StorageGetById(tv->storage, storage_type, id.id); +} + +int ThreadSetStorageById(ThreadVars *tv, ThreadStorageId id, void *ptr) +{ + return StorageSetById(tv->storage, storage_type, id.id, ptr); +} + +void *ThreadAllocStorageById(ThreadVars *tv, ThreadStorageId id) +{ + return StorageAllocByIdPrealloc(tv->storage, storage_type, id.id); +} + +void ThreadFreeStorageById(ThreadVars *tv, ThreadStorageId id) +{ + StorageFreeById(tv->storage, storage_type, id.id); +} + +void ThreadFreeStorage(ThreadVars *tv) +{ + if (ThreadStorageSize() > 0) + StorageFreeAll(tv->storage, storage_type); +} + +ThreadStorageId ThreadStorageRegister(const char *name, const unsigned int size, + void *(*Alloc)(unsigned int), void (*Free)(void *)) +{ + int id = StorageRegister(storage_type, name, size, Alloc, Free); + ThreadStorageId tsi = { .id = id }; + return tsi; +} + +#ifdef UNITTESTS + +static void *StorageTestAlloc(unsigned int size) +{ + return SCCalloc(1, size); +} + +static void StorageTestFree(void *x) +{ + SCFree(x); +} + +static int ThreadStorageTest01(void) +{ + StorageInit(); + + ThreadStorageId id1 = ThreadStorageRegister("test", 8, StorageTestAlloc, StorageTestFree); + FAIL_IF(id1.id < 0); + + ThreadStorageId id2 = ThreadStorageRegister("variable", 24, StorageTestAlloc, StorageTestFree); + FAIL_IF(id2.id < 0); + + ThreadStorageId id3 = + ThreadStorageRegister("store", sizeof(void *), StorageTestAlloc, StorageTestFree); + FAIL_IF(id3.id < 0); + + FAIL_IF(StorageFinalize() < 0); + + ThreadVars *tv = SCCalloc(1, sizeof(ThreadVars) + ThreadStorageSize()); + FAIL_IF_NULL(tv); + + void *ptr = ThreadGetStorageById(tv, id1); + FAIL_IF_NOT_NULL(ptr); + + ptr = ThreadGetStorageById(tv, id2); + FAIL_IF_NOT_NULL(ptr); + + ptr = ThreadGetStorageById(tv, id3); + FAIL_IF_NOT_NULL(ptr); + + void *ptr1a = ThreadAllocStorageById(tv, id1); + FAIL_IF_NULL(ptr1a); + + void *ptr2a = ThreadAllocStorageById(tv, id2); + FAIL_IF_NULL(ptr2a); + + void *ptr3a = ThreadAllocStorageById(tv, id3); + FAIL_IF_NULL(ptr3a); + + void *ptr1b = ThreadGetStorageById(tv, id1); + FAIL_IF(ptr1a != ptr1b); + + void *ptr2b = ThreadGetStorageById(tv, id2); + FAIL_IF(ptr2a != ptr2b); + + void *ptr3b = ThreadGetStorageById(tv, id3); + FAIL_IF(ptr3a != ptr3b); + + ThreadFreeStorage(tv); + StorageCleanup(); + SCFree(tv); + PASS; +} + +static int ThreadStorageTest02(void) +{ + StorageInit(); + + ThreadStorageId id1 = ThreadStorageRegister("test", sizeof(void *), NULL, StorageTestFree); + FAIL_IF(id1.id < 0); + + FAIL_IF(StorageFinalize() < 0); + + ThreadVars *tv = SCCalloc(1, sizeof(ThreadVars) + ThreadStorageSize()); + FAIL_IF_NULL(tv); + + void *ptr = ThreadGetStorageById(tv, id1); + FAIL_IF_NOT_NULL(ptr); + + void *ptr1a = SCMalloc(128); + FAIL_IF_NULL(ptr1a); + + ThreadSetStorageById(tv, id1, ptr1a); + + void *ptr1b = ThreadGetStorageById(tv, id1); + FAIL_IF(ptr1a != ptr1b); + + ThreadFreeStorage(tv); + StorageCleanup(); + PASS; +} + +static int ThreadStorageTest03(void) +{ + StorageInit(); + + ThreadStorageId id1 = ThreadStorageRegister("test1", sizeof(void *), NULL, StorageTestFree); + FAIL_IF(id1.id < 0); + + ThreadStorageId id2 = ThreadStorageRegister("test2", sizeof(void *), NULL, StorageTestFree); + FAIL_IF(id2.id < 0); + + ThreadStorageId id3 = ThreadStorageRegister("test3", 32, StorageTestAlloc, StorageTestFree); + FAIL_IF(id3.id < 0); + + FAIL_IF(StorageFinalize() < 0); + + ThreadVars *tv = SCCalloc(1, sizeof(ThreadVars) + ThreadStorageSize()); + FAIL_IF_NULL(tv); + + void *ptr = ThreadGetStorageById(tv, id1); + FAIL_IF_NOT_NULL(ptr); + + void *ptr1a = SCMalloc(128); + FAIL_IF_NULL(ptr1a); + + ThreadSetStorageById(tv, id1, ptr1a); + + void *ptr2a = SCMalloc(256); + FAIL_IF_NULL(ptr2a); + + ThreadSetStorageById(tv, id2, ptr2a); + + void *ptr3a = ThreadAllocStorageById(tv, id3); + FAIL_IF_NULL(ptr3a); + + void *ptr1b = ThreadGetStorageById(tv, id1); + FAIL_IF(ptr1a != ptr1b); + + void *ptr2b = ThreadGetStorageById(tv, id2); + FAIL_IF(ptr2a != ptr2b); + + void *ptr3b = ThreadGetStorageById(tv, id3); + FAIL_IF(ptr3a != ptr3b); + + ThreadFreeStorage(tv); + StorageCleanup(); + PASS; +} +#endif + +void RegisterThreadStorageTests(void) +{ +#ifdef UNITTESTS + UtRegisterTest("ThreadStorageTest01", ThreadStorageTest01); + UtRegisterTest("ThreadStorageTest02", ThreadStorageTest02); + UtRegisterTest("ThreadStorageTest03", ThreadStorageTest03); +#endif +} diff --git a/src/thread-storage.h b/src/thread-storage.h new file mode 100644 index 000000000000..5dd22570b0fa --- /dev/null +++ b/src/thread-storage.h @@ -0,0 +1,45 @@ +/* Copyright (C) 2024 Open Information Security Foundation + * + * You can copy, redistribute or modify this Program under the terms of + * the GNU General Public License version 2 as published by the Free + * Software Foundation. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * version 2 along with this program; if not, write to the Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA + * 02110-1301, USA. + */ + +/** + * Thread wrapper around storage API. + */ + +#ifndef SURICATA_THREAD_STORAGE_H +#define SURICATA_THREAD_STORAGE_H + +#include "threadvars.h" + +typedef struct ThreadStorageId { + int id; +} ThreadStorageId; + +unsigned int ThreadStorageSize(void); + +void *ThreadGetStorageById(const ThreadVars *tv, ThreadStorageId id); +int ThreadSetStorageById(ThreadVars *tv, ThreadStorageId id, void *ptr); +void *ThreadAllocStorageById(ThreadVars *tv, ThreadStorageId id); + +void ThreadFreeStorageById(ThreadVars *tv, ThreadStorageId id); +void ThreadFreeStorage(ThreadVars *tv); + +void RegisterThreadStorageTests(void); + +ThreadStorageId ThreadStorageRegister(const char *name, const unsigned int size, + void *(*Alloc)(unsigned int), void (*Free)(void *)); + +#endif /* SURICATA_THREAD_STORAGE_H */ diff --git a/src/threads.c b/src/threads.c index 1708a8f5cd37..919e6422e32f 100644 --- a/src/threads.c +++ b/src/threads.c @@ -25,6 +25,7 @@ */ #include "suricata-common.h" +#include "thread-storage.h" #include "util-unittest.h" #include "util-debug.h" #include "threads.h" @@ -149,5 +150,6 @@ void ThreadMacrosRegisterTests(void) UtRegisterTest("ThreadMacrosTest03RWLocks", ThreadMacrosTest03RWLocks); UtRegisterTest("ThreadMacrosTest04RWLocks", ThreadMacrosTest04RWLocks); // UtRegisterTest("ThreadMacrosTest05RWLocks", ThreadMacrosTest05RWLocks); + RegisterThreadStorageTests(); #endif /* UNIT TESTS */ } diff --git a/src/threadvars.h b/src/threadvars.h index cebcdb4e3ac1..6f339e9839d5 100644 --- a/src/threadvars.h +++ b/src/threadvars.h @@ -28,6 +28,7 @@ #include "counters.h" #include "packet-queue.h" #include "util-atomic.h" +#include "util-storage.h" struct TmSlot_; @@ -135,6 +136,7 @@ typedef struct ThreadVars_ { struct FlowQueue_ *flow_queue; bool break_loop; + Storage storage[]; } ThreadVars; /** Thread setup flags: */ diff --git a/src/tm-threads.c b/src/tm-threads.c index b0d0f8686ba0..c65995ad351b 100644 --- a/src/tm-threads.c +++ b/src/tm-threads.c @@ -30,6 +30,7 @@ #include "stream.h" #include "runmodes.h" #include "threadvars.h" +#include "thread-storage.h" #include "tm-queues.h" #include "tm-queuehandlers.h" #include "tm-threads.h" @@ -919,7 +920,7 @@ ThreadVars *TmThreadCreate(const char *name, const char *inq_name, const char *i SCLogDebug("creating thread \"%s\"...", name); /* XXX create separate function for this: allocate a thread container */ - tv = SCCalloc(1, sizeof(ThreadVars)); + tv = SCCalloc(1, sizeof(ThreadVars) + ThreadStorageSize()); if (unlikely(tv == NULL)) goto error; @@ -1577,6 +1578,8 @@ static void TmThreadFree(ThreadVars *tv) SCLogDebug("Freeing thread '%s'.", tv->name); + ThreadFreeStorage(tv); + if (tv->flow_queue) { BUG_ON(tv->flow_queue->qlen != 0); SCFree(tv->flow_queue); diff --git a/src/util-storage.c b/src/util-storage.c index 02f69a568cd2..bae251432315 100644 --- a/src/util-storage.c +++ b/src/util-storage.c @@ -59,6 +59,8 @@ static const char *StoragePrintType(StorageEnum type) return "ippair"; case STORAGE_DEVICE: return "livedevice"; + case STORAGE_THREAD: + return "thread"; case STORAGE_MAX: return "max"; } diff --git a/src/util-storage.h b/src/util-storage.h index 11d64bdbecbd..fce1f964eb14 100644 --- a/src/util-storage.h +++ b/src/util-storage.h @@ -31,6 +31,7 @@ typedef enum StorageEnum_ { STORAGE_FLOW, STORAGE_IPPAIR, STORAGE_DEVICE, + STORAGE_THREAD, STORAGE_MAX, } StorageEnum; From 27bbc0c84d55e3a82de70d29671ea58504443f73 Mon Sep 17 00:00:00 2001 From: Jason Ish Date: Fri, 11 Oct 2024 11:48:50 -0600 Subject: [PATCH 3/7] threads: add initialization callbacks For library users and plugins that need to hook into the thread life cycle, perhaps to initialize some thread storage. --- src/Makefile.am | 2 ++ src/thread-callbacks.c | 55 ++++++++++++++++++++++++++++++++++++++++++ src/thread-callbacks.h | 54 +++++++++++++++++++++++++++++++++++++++++ src/tm-threads.c | 3 +++ 4 files changed, 114 insertions(+) create mode 100644 src/thread-callbacks.c create mode 100644 src/thread-callbacks.h diff --git a/src/Makefile.am b/src/Makefile.am index a2308c0de34f..a269a8e82725 100755 --- a/src/Makefile.am +++ b/src/Makefile.am @@ -449,6 +449,7 @@ noinst_HEADERS = \ suricata-common.h \ suricata.h \ suricata-plugin.h \ + thread-callbacks.h \ thread-storage.h \ threads-debug.h \ threads.h \ @@ -1013,6 +1014,7 @@ libsuricata_c_a_SOURCES = \ stream-tcp-sack.c \ stream-tcp-util.c \ suricata.c \ + thread-callbacks.c \ thread-storage.c \ threads.c \ tm-modules.c \ diff --git a/src/thread-callbacks.c b/src/thread-callbacks.c new file mode 100644 index 000000000000..ede35d7107ce --- /dev/null +++ b/src/thread-callbacks.c @@ -0,0 +1,55 @@ +/* Copyright (C) 2024 Open Information Security Foundation + * + * You can copy, redistribute or modify this Program under the terms of + * the GNU General Public License version 2 as published by the Free + * Software Foundation. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * version 2 along with this program; if not, write to the Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA + * 02110-1301, USA. + */ + +#include "thread-callbacks.h" + +typedef struct ThreadInitCallback_ { + SCThreadInitCallbackFn Callback; + void *user; + struct ThreadInitCallback_ *next; +} ThreadInitCallback; + +static ThreadInitCallback *init_callbacks = NULL; + +bool SCThreadRegisterInitCallback(SCThreadInitCallbackFn fn, void *user) +{ + ThreadInitCallback *cb = SCCalloc(1, sizeof(*cb)); + if (cb == NULL) { + return false; + } + cb->Callback = fn; + cb->user = user; + if (init_callbacks == NULL) { + init_callbacks = cb; + } else { + ThreadInitCallback *current = init_callbacks; + while (current->next != NULL) { + current = current->next; + } + current->next = cb; + } + return true; +} + +void SCThreadRunInitCallbacks(ThreadVars *tv) +{ + ThreadInitCallback *cb = init_callbacks; + while (cb != NULL) { + cb->Callback(tv, cb->user); + cb = cb->next; + } +} diff --git a/src/thread-callbacks.h b/src/thread-callbacks.h new file mode 100644 index 000000000000..5bcd638bf86b --- /dev/null +++ b/src/thread-callbacks.h @@ -0,0 +1,54 @@ +/* Copyright (C) 2024 Open Information Security Foundation + * + * You can copy, redistribute or modify this Program under the terms of + * the GNU General Public License version 2 as published by the Free + * Software Foundation. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * version 2 along with this program; if not, write to the Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA + * 02110-1301, USA. + */ + +#ifndef SURICATA_THREAD_CALLBACKS_H +#define SURICATA_THREAD_CALLBACKS_H + +#include "suricata-common.h" +#include "threadvars.h" + +/** \brief Function type for thread intialization callbacks. + * + * Once registered by SCThreadRegisterInitCallback, this function will + * be called for every thread being initialized during Suricata + * startup. + * + * \param tv The ThreadVars struct that has just been initialized. + * \param user The user data provided when registering the callback. + */ +typedef void (*SCThreadInitCallbackFn)(ThreadVars *tv, void *user); + +/** \brief Register a thread init callback. + * + * Register a user provided function to be called every time a thread is + * initialized for use. + * + * \param fn Pointer to function to be called + * \param user Additional user data to be passed to callback + * + * \returns true if callback was registered, otherwise false if the + * callback could not be registered due to memory allocation error. + */ +bool SCThreadRegisterInitCallback(SCThreadInitCallbackFn fn, void *user); + +/** \internal + * + * Run all registered flow init callbacks. + */ +void SCThreadRunInitCallbacks(ThreadVars *tv); + +#endif /* SURICATA_THREAD_CALLBACKS_H */ diff --git a/src/tm-threads.c b/src/tm-threads.c index c65995ad351b..07f9a9390df0 100644 --- a/src/tm-threads.c +++ b/src/tm-threads.c @@ -29,6 +29,7 @@ #include "suricata.h" #include "stream.h" #include "runmodes.h" +#include "thread-callbacks.h" #include "threadvars.h" #include "thread-storage.h" #include "tm-queues.h" @@ -1012,6 +1013,8 @@ ThreadVars *TmThreadCreate(const char *name, const char *inq_name, const char *i if (mucond != 0) TmThreadInitMC(tv); + SCThreadRunInitCallbacks(tv); + return tv; error: From 73c499b6bdc1a0b5d91659d3e3a5a954e1fccae0 Mon Sep 17 00:00:00 2001 From: Jason Ish Date: Fri, 11 Oct 2024 13:21:14 -0600 Subject: [PATCH 4/7] eve: user callbacks for adding additional data Provide a way for library/plugin users to register a callback that will be called prior to an EVE record being closed. The callback will be passed ThreadVars, Packet, and Flow pointers if available, as well as private user data. --- src/output-eve-stream.c | 2 +- src/output-eve.c | 39 ++++++++++++++++++++++++++++++++++ src/output-eve.h | 43 ++++++++++++++++++++++++++++++++++++++ src/output-json-alert.c | 6 +++--- src/output-json-anomaly.c | 21 +++++++++---------- src/output-json-arp.c | 2 +- src/output-json-dcerpc.c | 2 +- src/output-json-dhcp.c | 2 +- src/output-json-dnp3.c | 4 ++-- src/output-json-dns.c | 8 +++---- src/output-json-drop.c | 6 +++--- src/output-json-file.c | 8 +++---- src/output-json-flow.c | 2 +- src/output-json-frame.c | 10 ++++----- src/output-json-http.c | 2 +- src/output-json-ike.c | 2 +- src/output-json-metadata.c | 2 +- src/output-json-mqtt.c | 2 +- src/output-json-netflow.c | 4 ++-- src/output-json-nfs.c | 2 +- src/output-json-pgsql.c | 2 +- src/output-json-smb.c | 2 +- src/output-json-smtp.c | 2 +- src/output-json-tls.c | 2 +- src/output-json.c | 5 ++++- src/output-json.h | 3 ++- src/output.c | 2 +- 27 files changed, 136 insertions(+), 51 deletions(-) diff --git a/src/output-eve-stream.c b/src/output-eve-stream.c index fcdf0c2e5c0b..4b44d86835e7 100644 --- a/src/output-eve-stream.c +++ b/src/output-eve-stream.c @@ -425,7 +425,7 @@ static int EveStreamLogger(ThreadVars *tv, void *thread_data, const Packet *p) /* Close stream. */ jb_close(js); - OutputJsonBuilderBuffer(js, td->ctx); + OutputJsonBuilderBuffer(tv, p, p->flow, js, td->ctx); jb_free(js); return TM_ECODE_OK; diff --git a/src/output-eve.c b/src/output-eve.c index d0d775cba7f7..2c67f3b6c38a 100644 --- a/src/output-eve.c +++ b/src/output-eve.c @@ -15,11 +15,50 @@ * 02110-1301, USA. */ +#include "suricata-common.h" #include "output-eve.h" #include "util-debug.h" +#include "rust.h" + +typedef struct EveUserCallback_ { + SCEveUserCallbackFn Callback; + void *user; + struct EveUserCallback_ *next; +} EveUserCallback; + +static EveUserCallback *eve_user_callbacks = NULL; static TAILQ_HEAD(, SCEveFileType_) output_types = TAILQ_HEAD_INITIALIZER(output_types); +bool SCEveRegisterCallback(SCEveUserCallbackFn fn, void *user) +{ + EveUserCallback *cb = SCCalloc(1, sizeof(*cb)); + if (cb == NULL) { + return false; + } + cb->Callback = fn; + cb->user = user; + if (eve_user_callbacks == NULL) { + eve_user_callbacks = cb; + } else { + EveUserCallback *current = eve_user_callbacks; + while (current->next != NULL) { + current = current->next; + } + current->next = cb; + } + return true; +} + +void SCEveRunCallbacks(ThreadVars *tv, const Packet *p, Flow *f, JsonBuilder *jb) +{ + EveUserCallback *cb = eve_user_callbacks; + while (cb != NULL) { + cb->Callback(tv, p, f, jb, cb->user); + cb = cb->next; + } +} + static bool IsBuiltinTypeName(const char *name) { const char *builtin[] = { diff --git a/src/output-eve.h b/src/output-eve.h index 7046c7b98005..7e55ce28f8e2 100644 --- a/src/output-eve.h +++ b/src/output-eve.h @@ -31,6 +31,7 @@ #define SURICATA_OUTPUT_EVE_H #include "suricata-common.h" +#include "rust.h" #include "conf.h" typedef uint32_t ThreadId; @@ -173,4 +174,46 @@ bool SCRegisterEveFileType(SCEveFileType *); SCEveFileType *SCEveFindFileType(const char *name); +/** \brief Function type for EVE callbacks. + * + * The function type for callbacks registered with + * SCEveRegisterCallback. This function will be called with the + * JsonBuilder just prior to the top-level object being closed. New + * fields maybe added, however there is no way to alter existing + * objects already added to the JsonBuilder. + * + * \param tv The ThreadVars for the thread performing the logging. + * \param p Packet if available. + * \param f Flow if available. + * \param user User data provided during callback registration. + */ +typedef void (*SCEveUserCallbackFn)( + ThreadVars *tv, const Packet *p, Flow *f, JsonBuilder *jb, void *user); + +/** \brief Register a callback for adding extra information to EVE logs. + * + * Allow users to register a callback for each EVE log. The callback + * is called just before the root object on the JsonBuilder is to be + * closed. + * + * New objects and fields can be append, but exist entries cannot be modified. + * + * Packet and Flow will be provided if available, but will other be + * NULL. + * + * Limitations: At this time the callbacks will only be called for EVE + * loggers that use JsonBuilder, notably this means it won't be called + * for stats records at this time. + * + * \returns true if callback is registered, false is not due to memory + * allocation error. + */ +bool SCEveRegisterCallback(SCEveUserCallbackFn fn, void *user); + +/** \internal + * + * Run EVE callbacks. + */ +void SCEveRunCallbacks(ThreadVars *tv, const Packet *p, Flow *f, JsonBuilder *jb); + #endif diff --git a/src/output-json-alert.c b/src/output-json-alert.c index 7822cc798045..91a55828a7a1 100644 --- a/src/output-json-alert.c +++ b/src/output-json-alert.c @@ -757,7 +757,7 @@ static int AlertJson(ThreadVars *tv, JsonAlertLogThread *aft, const Packet *p) EveAddVerdict(jb, p); } - OutputJsonBuilderBuffer(jb, aft->ctx); + OutputJsonBuilderBuffer(tv, p, p->flow, jb, aft->ctx); jb_free(jb); } @@ -767,7 +767,7 @@ static int AlertJson(ThreadVars *tv, JsonAlertLogThread *aft, const Packet *p) CreateEveHeader(p, LOG_DIR_PACKET, "packet", NULL, json_output_ctx->eve_ctx); if (unlikely(packetjs != NULL)) { EvePacket(p, packetjs, 0); - OutputJsonBuilderBuffer(packetjs, aft->ctx); + OutputJsonBuilderBuffer(tv, p, p->flow, packetjs, aft->ctx); jb_free(packetjs); } } @@ -801,7 +801,7 @@ static int AlertJsonDecoderEvent(ThreadVars *tv, JsonAlertLogThread *aft, const AlertJsonHeader(p, pa, jb, json_output_ctx->flags, NULL, NULL); - OutputJsonBuilderBuffer(jb, aft->ctx); + OutputJsonBuilderBuffer(tv, p, p->flow, jb, aft->ctx); jb_free(jb); } diff --git a/src/output-json-anomaly.c b/src/output-json-anomaly.c index 241cb974a758..00f82fa3685e 100644 --- a/src/output-json-anomaly.c +++ b/src/output-json-anomaly.c @@ -143,16 +143,16 @@ static int AnomalyDecodeEventJson(ThreadVars *tv, JsonAnomalyLogThread *aft, EvePacket(p, js, GET_PKT_LEN(p) < 32 ? GET_PKT_LEN(p) : 32); } - OutputJsonBuilderBuffer(js, aft->ctx); + OutputJsonBuilderBuffer(tv, p, p->flow, js, aft->ctx); jb_free(js); } return TM_ECODE_OK; } -static int AnomalyAppLayerDecoderEventJson(JsonAnomalyLogThread *aft, - const Packet *p, AppLayerDecoderEvents *decoder_events, - bool is_pktlayer, const char *layer, uint64_t tx_id) +static int AnomalyAppLayerDecoderEventJson(ThreadVars *tv, JsonAnomalyLogThread *aft, + const Packet *p, AppLayerDecoderEvents *decoder_events, bool is_pktlayer, const char *layer, + uint64_t tx_id) { const char *alprotoname = AppLayerGetProtoName(p->flow->alproto); @@ -201,7 +201,7 @@ static int AnomalyAppLayerDecoderEventJson(JsonAnomalyLogThread *aft, /* anomaly */ jb_close(js); - OutputJsonBuilderBuffer(js, aft->ctx); + OutputJsonBuilderBuffer(tv, p, p->flow, js, aft->ctx); jb_free(js); /* Current implementation assumes a single owner for this value */ @@ -223,8 +223,7 @@ static int JsonAnomalyTxLogger(ThreadVars *tv, void *thread_data, const Packet * decoder_events = AppLayerParserGetEventsByTx(f->proto, f->alproto, tx); if (decoder_events && decoder_events->event_last_logged < decoder_events->cnt) { SCLogDebug("state %p, tx: %p, tx_id: %"PRIu64, state, tx, tx_id); - AnomalyAppLayerDecoderEventJson(aft, p, decoder_events, false, - "proto_parser", tx_id); + AnomalyAppLayerDecoderEventJson(tv, aft, p, decoder_events, false, "proto_parser", tx_id); } return TM_ECODE_OK; } @@ -255,8 +254,8 @@ static int AnomalyJson(ThreadVars *tv, JsonAnomalyLogThread *aft, const Packet * if (aft->json_output_ctx->flags & LOG_JSON_APPLAYER_TYPE) { /* app layer proto detect events */ if (rc == TM_ECODE_OK && AnomalyHasPacketAppLayerEvents(p)) { - rc = AnomalyAppLayerDecoderEventJson(aft, p, p->app_layer_events, - true, "proto_detect", TX_ID_UNUSED); + rc = AnomalyAppLayerDecoderEventJson( + tv, aft, p, p->app_layer_events, true, "proto_detect", TX_ID_UNUSED); } /* parser state events */ @@ -264,8 +263,8 @@ static int AnomalyJson(ThreadVars *tv, JsonAnomalyLogThread *aft, const Packet * SCLogDebug("Checking for anomaly events; alproto %d", p->flow->alproto); AppLayerDecoderEvents *parser_events = AppLayerParserGetDecoderEvents(p->flow->alparser); if (parser_events && (parser_events->event_last_logged < parser_events->cnt)) { - rc = AnomalyAppLayerDecoderEventJson(aft, p, parser_events, - false, "parser", TX_ID_UNUSED); + rc = AnomalyAppLayerDecoderEventJson( + tv, aft, p, parser_events, false, "parser", TX_ID_UNUSED); } } } diff --git a/src/output-json-arp.c b/src/output-json-arp.c index 0490c6b54d1e..87a80d8cdb0b 100644 --- a/src/output-json-arp.c +++ b/src/output-json-arp.c @@ -90,7 +90,7 @@ static int JsonArpLogger(ThreadVars *tv, void *thread_data, const Packet *p) JSONFormatAndAddMACAddr(jb, "dest_mac", arph->dest_mac, false); jb_set_string(jb, "dest_ip", dstip); jb_close(jb); /* arp */ - OutputJsonBuilderBuffer(jb, thread); + OutputJsonBuilderBuffer(tv, p, p->flow, jb, thread); jb_free(jb); return TM_ECODE_OK; diff --git a/src/output-json-dcerpc.c b/src/output-json-dcerpc.c index 17e0199ed727..3b3bff90feac 100644 --- a/src/output-json-dcerpc.c +++ b/src/output-json-dcerpc.c @@ -47,7 +47,7 @@ static int JsonDCERPCLogger(ThreadVars *tv, void *thread_data, jb_close(jb); MemBufferReset(thread->buffer); - OutputJsonBuilderBuffer(jb, thread); + OutputJsonBuilderBuffer(tv, p, p->flow, jb, thread); jb_free(jb); return TM_ECODE_OK; diff --git a/src/output-json-dhcp.c b/src/output-json-dhcp.c index 9c7d9dff9230..a4a4a29990e8 100644 --- a/src/output-json-dhcp.c +++ b/src/output-json-dhcp.c @@ -72,7 +72,7 @@ static int JsonDHCPLogger(ThreadVars *tv, void *thread_data, rs_dhcp_logger_log(ctx->rs_logger, tx, js); - OutputJsonBuilderBuffer(js, thread->thread); + OutputJsonBuilderBuffer(tv, p, p->flow, js, thread->thread); jb_free(js); return TM_ECODE_OK; diff --git a/src/output-json-dnp3.c b/src/output-json-dnp3.c index 53cecd78a1aa..ea557ff206a8 100644 --- a/src/output-json-dnp3.c +++ b/src/output-json-dnp3.c @@ -246,7 +246,7 @@ static int JsonDNP3LoggerToServer(ThreadVars *tv, void *thread_data, jb_open_object(js, "dnp3"); JsonDNP3LogRequest(js, tx); jb_close(js); - OutputJsonBuilderBuffer(js, thread->ctx); + OutputJsonBuilderBuffer(tv, p, p->flow, js, thread->ctx); jb_free(js); SCReturnInt(TM_ECODE_OK); @@ -267,7 +267,7 @@ static int JsonDNP3LoggerToClient(ThreadVars *tv, void *thread_data, jb_open_object(js, "dnp3"); JsonDNP3LogResponse(js, tx); jb_close(js); - OutputJsonBuilderBuffer(js, thread->ctx); + OutputJsonBuilderBuffer(tv, p, p->flow, js, thread->ctx); jb_free(js); SCReturnInt(TM_ECODE_OK); diff --git a/src/output-json-dns.c b/src/output-json-dns.c index 3954da2336dc..cb60a4509a32 100644 --- a/src/output-json-dns.c +++ b/src/output-json-dns.c @@ -331,7 +331,7 @@ static int JsonDoh2Logger(ThreadVars *tv, void *thread_data, const Packet *p, Fl } out: if (r || r2) { - OutputJsonBuilderBuffer(jb, td->ctx); + OutputJsonBuilderBuffer(tv, p, p->flow, jb, td->ctx); } jb_free(jb); return TM_ECODE_OK; @@ -363,7 +363,7 @@ static int JsonDnsLoggerToServer(ThreadVars *tv, void *thread_data, } jb_close(jb); - OutputJsonBuilderBuffer(jb, td->ctx); + OutputJsonBuilderBuffer(tv, p, p->flow, jb, td->ctx); jb_free(jb); } @@ -392,7 +392,7 @@ static int JsonDnsLoggerToClient(ThreadVars *tv, void *thread_data, jb_set_int(jb, "version", 2); SCDnsLogJsonAnswer(txptr, td->dnslog_ctx->flags, jb); jb_close(jb); - OutputJsonBuilderBuffer(jb, td->ctx); + OutputJsonBuilderBuffer(tv, p, p->flow, jb, td->ctx); jb_free(jb); } @@ -432,7 +432,7 @@ static int JsonDnsLogger(ThreadVars *tv, void *thread_data, const Packet *p, Flo } if (SCDnsLogJson(txptr, td->dnslog_ctx->flags, jb)) { - OutputJsonBuilderBuffer(jb, td->ctx); + OutputJsonBuilderBuffer(tv, p, p->flow, jb, td->ctx); } jb_free(jb); } diff --git a/src/output-json-drop.c b/src/output-json-drop.c index b82c632daf65..1ac27a209d2a 100644 --- a/src/output-json-drop.c +++ b/src/output-json-drop.c @@ -85,7 +85,7 @@ static int g_droplog_flows_start = 1; * * \return return TM_ECODE_OK on success */ -static int DropLogJSON (JsonDropLogThread *aft, const Packet *p) +static int DropLogJSON(ThreadVars *tv, JsonDropLogThread *aft, const Packet *p) { JsonDropOutputCtx *drop_ctx = aft->drop_ctx; @@ -191,7 +191,7 @@ static int DropLogJSON (JsonDropLogThread *aft, const Packet *p) } } - OutputJsonBuilderBuffer(js, aft->ctx); + OutputJsonBuilderBuffer(tv, p, p->flow, js, aft->ctx); jb_free(js); return TM_ECODE_OK; @@ -326,7 +326,7 @@ static OutputInitResult JsonDropLogInitCtxSub(ConfNode *conf, OutputCtx *parent_ static int JsonDropLogger(ThreadVars *tv, void *thread_data, const Packet *p) { JsonDropLogThread *td = thread_data; - int r = DropLogJSON(td, p); + int r = DropLogJSON(tv, td, p); if (r < 0) return -1; diff --git a/src/output-json-file.c b/src/output-json-file.c index 509ae488bbee..e1f33893806e 100644 --- a/src/output-json-file.c +++ b/src/output-json-file.c @@ -213,8 +213,8 @@ JsonBuilder *JsonBuildFileInfoRecord(const Packet *p, const File *ff, void *tx, * \internal * \brief Write meta data on a single line json record */ -static void FileWriteJsonRecord(JsonFileLogThread *aft, const Packet *p, const File *ff, void *tx, - const uint64_t tx_id, uint8_t dir, OutputJsonCtx *eve_ctx) +static void FileWriteJsonRecord(ThreadVars *tv, JsonFileLogThread *aft, const Packet *p, + const File *ff, void *tx, const uint64_t tx_id, uint8_t dir, OutputJsonCtx *eve_ctx) { HttpXFFCfg *xff_cfg = aft->filelog_ctx->xff_cfg != NULL ? aft->filelog_ctx->xff_cfg : aft->filelog_ctx->parent_xff_cfg; @@ -223,7 +223,7 @@ static void FileWriteJsonRecord(JsonFileLogThread *aft, const Packet *p, const F return; } - OutputJsonBuilderBuffer(js, aft->ctx); + OutputJsonBuilderBuffer(tv, p, p->flow, js, aft->ctx); jb_free(js); } @@ -237,7 +237,7 @@ static int JsonFileLogger(ThreadVars *tv, void *thread_data, const Packet *p, co SCLogDebug("ff %p", ff); - FileWriteJsonRecord(aft, p, ff, tx, tx_id, dir, aft->filelog_ctx->eve_ctx); + FileWriteJsonRecord(tv, aft, p, ff, tx, tx_id, dir, aft->filelog_ctx->eve_ctx); return 0; } diff --git a/src/output-json-flow.c b/src/output-json-flow.c index f7826734f0cb..051d530fb1ef 100644 --- a/src/output-json-flow.c +++ b/src/output-json-flow.c @@ -340,7 +340,7 @@ static int JsonFlowLogger(ThreadVars *tv, void *thread_data, Flow *f) EveFlowLogJSON(thread, jb, f); - OutputJsonBuilderBuffer(jb, thread); + OutputJsonBuilderBuffer(tv, NULL, f, jb, thread); jb_free(jb); SCReturnInt(TM_ECODE_OK); diff --git a/src/output-json-frame.c b/src/output-json-frame.c index 09ec4aaab110..90224240f43f 100644 --- a/src/output-json-frame.c +++ b/src/output-json-frame.c @@ -287,8 +287,8 @@ void FrameJsonLogOneFrame(const uint8_t ipproto, const Frame *frame, Flow *f, jb_close(jb); } -static int FrameJsonUdp( - JsonFrameLogThread *aft, const Packet *p, Flow *f, FramesContainer *frames_container) +static int FrameJsonUdp(ThreadVars *tv, JsonFrameLogThread *aft, const Packet *p, Flow *f, + FramesContainer *frames_container) { FrameJsonOutputCtx *json_output_ctx = aft->json_output_ctx; @@ -315,7 +315,7 @@ static int FrameJsonUdp( jb_set_string(jb, "app_proto", AppProtoToString(f->alproto)); FrameJsonLogOneFrame(IPPROTO_UDP, frame, p->flow, NULL, p, jb, aft->payload_buffer); - OutputJsonBuilderBuffer(jb, aft->ctx); + OutputJsonBuilderBuffer(tv, p, p->flow, jb, aft->ctx); jb_free(jb); frame->flags |= FRAME_FLAG_LOGGED; } @@ -333,7 +333,7 @@ static int FrameJson(ThreadVars *tv, JsonFrameLogThread *aft, const Packet *p) return TM_ECODE_OK; if (p->proto == IPPROTO_UDP) { - return FrameJsonUdp(aft, p, p->flow, frames_container); + return FrameJsonUdp(tv, aft, p, p->flow, frames_container); } BUG_ON(p->proto != IPPROTO_TCP); @@ -387,7 +387,7 @@ static int FrameJson(ThreadVars *tv, JsonFrameLogThread *aft, const Packet *p) jb_set_string(jb, "app_proto", AppProtoToString(p->flow->alproto)); FrameJsonLogOneFrame(IPPROTO_TCP, frame, p->flow, stream, p, jb, aft->payload_buffer); - OutputJsonBuilderBuffer(jb, aft->ctx); + OutputJsonBuilderBuffer(tv, p, p->flow, jb, aft->ctx); jb_free(jb); frame->flags |= FRAME_FLAG_LOGGED; } else if (frame != NULL) { diff --git a/src/output-json-http.c b/src/output-json-http.c index 0c5b875ee9ad..b45be9a45b6b 100644 --- a/src/output-json-http.c +++ b/src/output-json-http.c @@ -493,7 +493,7 @@ static int JsonHttpLogger(ThreadVars *tv, void *thread_data, const Packet *p, Fl } } - OutputJsonBuilderBuffer(js, jhl->ctx); + OutputJsonBuilderBuffer(tv, p, p->flow, js, jhl->ctx); jb_free(js); SCReturnInt(TM_ECODE_OK); diff --git a/src/output-json-ike.c b/src/output-json-ike.c index 470026fde13b..a13ef0e1d944 100644 --- a/src/output-json-ike.c +++ b/src/output-json-ike.c @@ -90,7 +90,7 @@ static int JsonIKELogger(ThreadVars *tv, void *thread_data, const Packet *p, Flo goto error; } - OutputJsonBuilderBuffer(jb, thread->ctx); + OutputJsonBuilderBuffer(tv, p, p->flow, jb, thread->ctx); jb_free(jb); return TM_ECODE_OK; diff --git a/src/output-json-metadata.c b/src/output-json-metadata.c index f97547c551b4..2602e4b9b3ef 100644 --- a/src/output-json-metadata.c +++ b/src/output-json-metadata.c @@ -74,7 +74,7 @@ static int MetadataJson(ThreadVars *tv, OutputJsonThreadCtx *aft, const Packet * if (!aft->ctx->cfg.include_metadata) { EveAddMetadata(p, p->flow, js); } - OutputJsonBuilderBuffer(js, aft); + OutputJsonBuilderBuffer(tv, p, p->flow, js, aft); jb_free(js); return TM_ECODE_OK; diff --git a/src/output-json-mqtt.c b/src/output-json-mqtt.c index 66cf67a0334b..c912ddcc1835 100644 --- a/src/output-json-mqtt.c +++ b/src/output-json-mqtt.c @@ -85,7 +85,7 @@ static int JsonMQTTLogger(ThreadVars *tv, void *thread_data, if (!rs_mqtt_logger_log(tx, thread->mqttlog_ctx->flags, thread->mqttlog_ctx->max_log_len, js)) goto error; - OutputJsonBuilderBuffer(js, thread->ctx); + OutputJsonBuilderBuffer(tv, p, p->flow, js, thread->ctx); jb_free(js); return TM_ECODE_OK; diff --git a/src/output-json-netflow.c b/src/output-json-netflow.c index 2e359bb909c5..e448ecd33bc4 100644 --- a/src/output-json-netflow.c +++ b/src/output-json-netflow.c @@ -276,7 +276,7 @@ static int JsonNetFlowLogger(ThreadVars *tv, void *thread_data, Flow *f) return TM_ECODE_OK; NetFlowLogEveToServer(jb, f); EveAddCommonOptions(&jhl->ctx->cfg, NULL, f, jb, LOG_DIR_FLOW_TOSERVER); - OutputJsonBuilderBuffer(jb, jhl); + OutputJsonBuilderBuffer(tv, NULL, f, jb, jhl); jb_free(jb); /* only log a response record if we actually have seen response packets */ @@ -286,7 +286,7 @@ static int JsonNetFlowLogger(ThreadVars *tv, void *thread_data, Flow *f) return TM_ECODE_OK; NetFlowLogEveToClient(jb, f); EveAddCommonOptions(&jhl->ctx->cfg, NULL, f, jb, LOG_DIR_FLOW_TOCLIENT); - OutputJsonBuilderBuffer(jb, jhl); + OutputJsonBuilderBuffer(tv, NULL, f, jb, jhl); jb_free(jb); } SCReturnInt(TM_ECODE_OK); diff --git a/src/output-json-nfs.c b/src/output-json-nfs.c index 72274a6b7865..0b08c0e5105d 100644 --- a/src/output-json-nfs.c +++ b/src/output-json-nfs.c @@ -94,7 +94,7 @@ static int JsonNFSLogger(ThreadVars *tv, void *thread_data, jb_close(jb); MemBufferReset(thread->buffer); - OutputJsonBuilderBuffer(jb, thread); + OutputJsonBuilderBuffer(tv, p, p->flow, jb, thread); jb_free(jb); return TM_ECODE_OK; } diff --git a/src/output-json-pgsql.c b/src/output-json-pgsql.c index 71bcd10f071d..9cba28d25d4e 100644 --- a/src/output-json-pgsql.c +++ b/src/output-json-pgsql.c @@ -80,7 +80,7 @@ static int JsonPgsqlLogger(ThreadVars *tv, void *thread_data, const Packet *p, F goto error; } - OutputJsonBuilderBuffer(jb, thread->ctx); + OutputJsonBuilderBuffer(tv, p, p->flow, jb, thread->ctx); jb_free(jb); return TM_ECODE_OK; diff --git a/src/output-json-smb.c b/src/output-json-smb.c index 279ee772e8f0..4be1fce93e72 100644 --- a/src/output-json-smb.c +++ b/src/output-json-smb.c @@ -59,7 +59,7 @@ static int JsonSMBLogger(ThreadVars *tv, void *thread_data, } jb_close(jb); - OutputJsonBuilderBuffer(jb, thread); + OutputJsonBuilderBuffer(tv, p, p->flow, jb, thread); jb_free(jb); return TM_ECODE_OK; diff --git a/src/output-json-smtp.c b/src/output-json-smtp.c index bddbc4a9fcc4..592645cb3c09 100644 --- a/src/output-json-smtp.c +++ b/src/output-json-smtp.c @@ -85,7 +85,7 @@ static int JsonSmtpLogger(ThreadVars *tv, void *thread_data, const Packet *p, Fl jb_close(jb); EveEmailLogJson(jhl, jb, p, f, state, tx, tx_id); - OutputJsonBuilderBuffer(jb, jhl->ctx); + OutputJsonBuilderBuffer(tv, p, p->flow, jb, jhl->ctx); jb_free(jb); diff --git a/src/output-json-tls.c b/src/output-json-tls.c index aa24b3380a0d..c4ba0e249e62 100644 --- a/src/output-json-tls.c +++ b/src/output-json-tls.c @@ -501,7 +501,7 @@ static int JsonTlsLogger(ThreadVars *tv, void *thread_data, const Packet *p, /* Close the tls object. */ jb_close(js); - OutputJsonBuilderBuffer(js, aft->ctx); + OutputJsonBuilderBuffer(tv, p, p->flow, js, aft->ctx); jb_free(js); return 0; diff --git a/src/output-json.c b/src/output-json.c index 1f411cc110b8..18376fd428a5 100644 --- a/src/output-json.c +++ b/src/output-json.c @@ -955,7 +955,8 @@ int OutputJSONBuffer(json_t *js, LogFileCtx *file_ctx, MemBuffer **buffer) return 0; } -int OutputJsonBuilderBuffer(JsonBuilder *js, OutputJsonThreadCtx *ctx) +int OutputJsonBuilderBuffer( + ThreadVars *tv, const Packet *p, Flow *f, JsonBuilder *js, OutputJsonThreadCtx *ctx) { LogFileCtx *file_ctx = ctx->file_ctx; MemBuffer **buffer = &ctx->buffer; @@ -967,6 +968,8 @@ int OutputJsonBuilderBuffer(JsonBuilder *js, OutputJsonThreadCtx *ctx) jb_set_string(js, "pcap_filename", PcapFileGetFilename()); } + SCEveRunCallbacks(tv, p, f, js); + jb_close(js); MemBufferReset(*buffer); diff --git a/src/output-json.h b/src/output-json.h index 761064f7e10a..89597e616a0f 100644 --- a/src/output-json.h +++ b/src/output-json.h @@ -103,7 +103,8 @@ JsonBuilder *CreateEveHeader(const Packet *p, enum OutputJsonLogDirection dir, JsonBuilder *CreateEveHeaderWithTxId(const Packet *p, enum OutputJsonLogDirection dir, const char *event_type, JsonAddrInfo *addr, uint64_t tx_id, OutputJsonCtx *eve_ctx); int OutputJSONBuffer(json_t *js, LogFileCtx *file_ctx, MemBuffer **buffer); -int OutputJsonBuilderBuffer(JsonBuilder *js, OutputJsonThreadCtx *ctx); +int OutputJsonBuilderBuffer( + ThreadVars *tv, const Packet *p, Flow *f, JsonBuilder *js, OutputJsonThreadCtx *ctx); OutputInitResult OutputJsonInitCtx(ConfNode *); OutputInitResult OutputJsonLogInitSub(ConfNode *conf, OutputCtx *parent_ctx); diff --git a/src/output.c b/src/output.c index 002f33b5abc6..b99897509c0f 100644 --- a/src/output.c +++ b/src/output.c @@ -927,7 +927,7 @@ static int JsonGenericLogger(ThreadVars *tv, void *thread_data, const Packet *p, goto error; } - OutputJsonBuilderBuffer(js, thread); + OutputJsonBuilderBuffer(tv, p, p->flow, js, thread); jb_free(js); return TM_ECODE_OK; From ca753fa4390f0fab211cb0794266a2554ca56bca Mon Sep 17 00:00:00 2001 From: Jason Ish Date: Tue, 15 Oct 2024 17:06:35 -0600 Subject: [PATCH 5/7] examples/plugin: update to find generated rust header Needed for changes to output-eve.h. --- examples/plugins/c-json-filetype/Makefile.in | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/examples/plugins/c-json-filetype/Makefile.in b/examples/plugins/c-json-filetype/Makefile.in index 0d4ec381ee4a..4d85c8ee2d59 100644 --- a/examples/plugins/c-json-filetype/Makefile.in +++ b/examples/plugins/c-json-filetype/Makefile.in @@ -5,7 +5,7 @@ # But as this is an example in the Suricata source tree we'll look for # includes in the source tree. -CPPFLAGS += -I@top_srcdir@/src -DHAVE_CONFIG_H +CPPFLAGS += -I@top_srcdir@/src -I@top_srcdir@/rust/gen -I@top_srcdir@/rust/dist -DHAVE_CONFIG_H # Currently the Suricata logging system requires this to be even for # plugins. From d07bd8e6a649bb97f4ffc893d02e57263c757855 Mon Sep 17 00:00:00 2001 From: Jason Ish Date: Thu, 17 Oct 2024 13:16:38 -0600 Subject: [PATCH 6/7] detect: break apart sigtable setup and initialization Allows initialization to be done early, so the table is ready for dynamic registration by plugins which are loaded before signature setup. --- src/detect-engine-register.c | 6 ++++-- src/detect-engine-register.h | 1 + src/runmode-unittests.c | 1 + src/suricata.c | 2 ++ 4 files changed, 8 insertions(+), 2 deletions(-) diff --git a/src/detect-engine-register.c b/src/detect-engine-register.c index 6ea3698c2c93..f36312b6cc57 100644 --- a/src/detect-engine-register.c +++ b/src/detect-engine-register.c @@ -450,7 +450,7 @@ void SigTableCleanup(void) } } -void SigTableSetup(void) +void SigTableInit(void) { if (sigmatch_table == NULL) { DETECT_TBLSIZE = DETECT_TBLSIZE_STATIC + DETECT_TBLSIZE_STEP; @@ -458,10 +458,12 @@ void SigTableSetup(void) if (sigmatch_table == NULL) { DETECT_TBLSIZE = 0; FatalError("Could not allocate sigmatch_table"); - return; } } +} +void SigTableSetup(void) +{ DetectSidRegister(); DetectPriorityRegister(); DetectPrefilterRegister(); diff --git a/src/detect-engine-register.h b/src/detect-engine-register.h index c9134c77b83a..db4cd957af9d 100644 --- a/src/detect-engine-register.h +++ b/src/detect-engine-register.h @@ -338,6 +338,7 @@ extern int DETECT_TBLSIZE_IDX; #define DETECT_TBLSIZE_STEP 256 int SigTableList(const char *keyword); void SigTableCleanup(void); +void SigTableInit(void); void SigTableSetup(void); void SigTableRegisterTests(void); diff --git a/src/runmode-unittests.c b/src/runmode-unittests.c index e116e86d5be5..35780ab101a0 100644 --- a/src/runmode-unittests.c +++ b/src/runmode-unittests.c @@ -240,6 +240,7 @@ void RunUnittests(int list_unittests, const char *regex_arg) AppLayerSetup(); /* hardcoded initialization code */ + SigTableInit(); SigTableSetup(); /* load the rule keywords */ TmqhSetup(); diff --git a/src/suricata.c b/src/suricata.c index 49505f94ba89..c3c05431606e 100644 --- a/src/suricata.c +++ b/src/suricata.c @@ -2668,6 +2668,8 @@ int PostConfLoadedSetup(SCInstance *suri) MacSetRegisterFlowStorage(); + SigTableInit(); + #ifdef HAVE_PLUGINS SCPluginsLoad(suri->capture_plugin_name, suri->capture_plugin_args); #endif From 239106b31f645f5369f85f932333c2adf402c279 Mon Sep 17 00:00:00 2001 From: Jason Ish Date: Fri, 18 Oct 2024 08:46:42 -0600 Subject: [PATCH 7/7] make: install-headers: rust-bindings.h rust-bindings.h was not being installed with "make install-headers", and its now pulled in by a header used for plugin support, so make sure its installed. We first attempt to install the "dist" version if exists, otherwise install the "gen" one. Also install the "gen" even if the "dist" one exists, as its going to be newer. --- src/Makefile.am | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/src/Makefile.am b/src/Makefile.am index a269a8e82725..709f82a5ce20 100755 --- a/src/Makefile.am +++ b/src/Makefile.am @@ -1197,6 +1197,12 @@ install-headers: for header in $(noinst_HEADERS); do \ $(INSTALL_DATA) $$header "$(DESTDIR)${includedir}/suricata"; \ done + if test -e ../rust/dist/rust-bindings.h; then \ + $(INSTALL_DATA) ../rust/dist/rust-bindings.h "$(DESTDIR)${includedir}/suricata"; \ + fi + if test -e ../rust/gen/rust-bindings.h; then \ + $(INSTALL_DATA) ../rust/gen/rust-bindings.h "$(DESTDIR)${includedir}/suricata"; \ + fi # Until we can remove autoconf.h from our headers, we need to to # provide this for library/plugin users.