refactor jsonl connector

fkie-cad · Aug 12, 2024 · 78d4476 · 78d4476
1 parent 7afd24e
commit 78d4476
Show file tree

Hide file tree

Showing 2 changed files with 3 additions and 7 deletions.
diff --git a/examples/exampledata/config/http_pipeline.yml b/examples/exampledata/config/http_pipeline.yml
@@ -24,7 +24,7 @@ metrics:
 input:
   httpinput:
     type: http_input
-    message_backlog_size: 150
+    message_backlog_size: 15000000
     collect_meta: True
     metafield_name: "@metadata"
     uvicorn_config:

diff --git a/logprep/connector/http/input.py b/logprep/connector/http/input.py
@@ -275,13 +275,9 @@ async def __call__(self, req, resp, **kwargs):  # pylint: disable=arguments-diff
         """jsonl endpoint method"""
         self.collect_metrics()
         data = await self.get_data(req)
-        data = data.decode("utf8")
-        event = kwargs.get("metadata", {})
         metadata = kwargs.get("metadata", {})
-        stripped_lines = map(str.strip, data.splitlines())
-        events = (self._decoder.decode(line) for line in stripped_lines if line)
-        for event in events:
-            self.messages.put({**event, **metadata}, block=False)
+        for event in self._decoder.decode_lines(data):
+            self.messages.put(event | metadata, block=False)
 
 
 class PlaintextHttpEndpoint(HttpEndpoint):