Skip to content

Commit

Permalink
Add flush timeout to s3 connector
Browse files Browse the repository at this point in the history
  • Loading branch information
saegel committed Mar 20, 2024
1 parent 31188ed commit 7ff807c
Showing 1 changed file with 4 additions and 0 deletions.
4 changes: 4 additions & 0 deletions logprep/connector/s3/output.py
Original file line number Diff line number Diff line change
Expand Up @@ -112,6 +112,9 @@ class Config(Output.Config):
)
"""The input callback is called after the maximum backlog size has been reached
if this is set to True (optional)"""
flush_timeout: Optional[int] = field(validator=validators.instance_of(int), default=60)
"""(Optional) Timout after :code:`message_backlog` is flushed if
:code:`message_backlog_size` is not reached."""

@define(kw_only=True)
class Metrics(Output.Metrics):
Expand Down Expand Up @@ -141,6 +144,7 @@ def __init__(self, name: str, configuration: "S3Output.Config", logger: Logger):
self._base_prefix = f"{self._config.base_prefix}/" if self._config.base_prefix else ""
self._s3_resource = None
self._setup_s3_resource()
self._schedule_task(task=self._write_backlog, seconds=self._config.flush_timeout)

def _setup_s3_resource(self):
session = boto3.Session(
Expand Down

0 comments on commit 7ff807c

Please sign in to comment.