From 420eb478e4015036d23c869c521ea75b91761b5b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Th=C3=A9o=20Bertin?= Date: Wed, 20 Mar 2024 14:49:29 +0100 Subject: [PATCH] LOGOM::ADDED:: Add the ability to define the spooling directory for DA queues --- CHANGELOG | 1 + vulture_os/applications/logfwd/form.py | 28 +++++++++++++------ vulture_os/applications/logfwd/models.py | 9 ++++++ .../apps/logfwd_LogOMElasticSearch.html | 11 ++++++++ .../templates/apps/logfwd_LogOMFWD.html | 11 ++++++++ .../templates/apps/logfwd_LogOMFile.html | 11 ++++++++ .../templates/apps/logfwd_LogOMHIREDIS.html | 11 ++++++++ .../templates/apps/logfwd_LogOMKAFKA.html | 11 ++++++++ .../templates/apps/logfwd_LogOMMongoDB.html | 11 ++++++++ .../templates/apps/logfwd_LogOMRELP.html | 11 ++++++++ 10 files changed, 106 insertions(+), 9 deletions(-) diff --git a/CHANGELOG b/CHANGELOG index 1f8b077a..b917b5ca 100644 --- a/CHANGELOG +++ b/CHANGELOG @@ -9,6 +9,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ### Added - [LOG_FORWARDER] [REDIS] Support for queues, sets, publish and streams modes for log insertion - [LOGOMELASTICSEARCH] Handle log insertion errors returned by elasticsearch +- [LOGOM] Add the ability to define the spooling directory for DA queues ### Changed - [DEPENDENCIES] Upgrade djongo and code for pymongo>=4 - [LOGOM] Code and logic factorization diff --git a/vulture_os/applications/logfwd/form.py b/vulture_os/applications/logfwd/form.py index be77eb59..c381c982 100644 --- a/vulture_os/applications/logfwd/form.py +++ b/vulture_os/applications/logfwd/form.py @@ -50,7 +50,8 @@ class Meta: model = LogOM fields = ('name', 'enabled', 'send_as_raw', 'queue_size', 'dequeue_size', 'queue_timeout_shutdown', 'max_workers', 'new_worker_minimum_messages', 'worker_timeout_shutdown', 'enable_retry', - 'enable_disk_assist', 'high_watermark', 'low_watermark', 'max_file_size', 'max_disk_space') + 'enable_disk_assist', 'high_watermark', 'low_watermark', 'max_file_size', 'max_disk_space', + 'spool_directory') widgets = { 'enabled': CheckboxInput(attrs={"class": " js-switch"}), @@ -68,11 +69,12 @@ class Meta: 'low_watermark': NumberInput(attrs={'class': 'form-control'}), 'max_file_size': NumberInput(attrs={'class': 'form-control'}), 'max_disk_space': NumberInput(attrs={'class': 'form-control'}), + 'spool_directory': TextInput(attrs={'class': 'form-control'}), } def __init__(self, *args, **kwargs): super().__init__(*args, **kwargs) - for field_name in ['high_watermark', 'low_watermark', 'max_file_size', 'max_disk_space']: + for field_name in ['high_watermark', 'low_watermark', 'max_file_size', 'max_disk_space', 'spool_directory']: self.fields[field_name].required = False def clean_name(self): @@ -109,7 +111,7 @@ class Meta: fields = ('name', 'enabled', 'file', 'flush_interval', 'async_writing', 'send_as_raw', 'retention_time', 'rotation_period', 'queue_size', 'dequeue_size', 'queue_timeout_shutdown', 'max_workers', 'new_worker_minimum_messages', 'worker_timeout_shutdown', 'enable_retry', 'enable_disk_assist', - 'high_watermark', 'low_watermark', 'max_file_size', 'max_disk_space') + 'high_watermark', 'low_watermark', 'max_file_size', 'max_disk_space', 'spool_directory') widgets = { 'enabled': CheckboxInput(attrs={"class": " js-switch"}), @@ -132,6 +134,7 @@ class Meta: 'low_watermark': NumberInput(attrs={'class': 'form-control'}), 'max_file_size': NumberInput(attrs={'class': 'form-control'}), 'max_disk_space': NumberInput(attrs={'class': 'form-control'}), + 'spool_directory': TextInput(attrs={'class': 'form-control'}), } def clean_name(self): @@ -154,7 +157,7 @@ class Meta: fields = ('name', 'enabled', 'target', 'port', 'tls_enabled', 'x509_certificate', 'send_as_raw', 'queue_size', 'dequeue_size', 'queue_timeout_shutdown', 'max_workers', 'new_worker_minimum_messages', 'worker_timeout_shutdown', 'enable_retry', 'enable_disk_assist', 'high_watermark', 'low_watermark', - 'max_file_size', 'max_disk_space') + 'max_file_size', 'max_disk_space', 'spool_directory') widgets = { 'enabled': CheckboxInput(attrs={"class": " js-switch"}), @@ -176,6 +179,7 @@ class Meta: 'low_watermark': NumberInput(attrs={'class': 'form-control'}), 'max_file_size': NumberInput(attrs={'class': 'form-control'}), 'max_disk_space': NumberInput(attrs={'class': 'form-control'}), + 'spool_directory': TextInput(attrs={'class': 'form-control'}), } def __init__(self, *args, **kwargs): @@ -197,7 +201,7 @@ class Meta: 'expire_key', 'stream_outfield', 'stream_capacitylimit', 'send_as_raw', 'queue_size', 'dequeue_size', 'queue_timeout_shutdown', 'max_workers', 'new_worker_minimum_messages', 'worker_timeout_shutdown', 'enable_retry', 'enable_disk_assist', 'high_watermark', 'low_watermark', - 'max_file_size', 'max_disk_space') + 'max_file_size', 'max_disk_space', 'spool_directory') widgets = { 'enabled': CheckboxInput(attrs={'class': 'js-switch'}), @@ -225,6 +229,7 @@ class Meta: 'low_watermark': NumberInput(attrs={'class': 'form-control'}), 'max_file_size': NumberInput(attrs={'class': 'form-control'}), 'max_disk_space': NumberInput(attrs={'class': 'form-control'}), + 'spool_directory': TextInput(attrs={'class': 'form-control'}), } def clean_key(self): @@ -250,7 +255,7 @@ class Meta: fields = ('name', 'enabled', 'target', 'port', 'protocol', 'zip_level', 'queue_size', 'dequeue_size', 'queue_timeout_shutdown', 'max_workers', 'new_worker_minimum_messages', 'worker_timeout_shutdown', 'enable_retry', 'enable_disk_assist', 'high_watermark', 'low_watermark', 'max_file_size', - 'max_disk_space', 'ratelimit_interval', 'ratelimit_burst', 'send_as_raw') + 'max_disk_space', 'spool_directory', 'ratelimit_interval', 'ratelimit_burst', 'send_as_raw') widgets = { 'enabled': CheckboxInput(attrs={"class": " js-switch"}), @@ -271,6 +276,7 @@ class Meta: 'low_watermark': NumberInput(attrs={'class': 'form-control'}), 'max_file_size': NumberInput(attrs={'class': 'form-control'}), 'max_disk_space': NumberInput(attrs={'class': 'form-control'}), + 'spool_directory': TextInput(attrs={'class': 'form-control'}), 'ratelimit_interval': NumberInput(attrs={'class': 'form-control'}), 'ratelimit_burst': NumberInput(attrs={'class': 'form-control'}), 'send_as_raw': CheckboxInput(attrs={'class': 'form-control js-switch'}) @@ -301,7 +307,8 @@ class Meta: fields = ('name', 'enabled', 'servers', 'es8_compatibility', 'data_stream_mode', 'retry_on_els_failures', 'index_pattern', 'uid', 'pwd', 'x509_certificate', 'send_as_raw', 'queue_size', 'dequeue_size', 'queue_timeout_shutdown', 'max_workers', 'new_worker_minimum_messages', 'worker_timeout_shutdown', 'enable_retry', - 'enable_disk_assist', 'high_watermark', 'low_watermark', 'max_file_size', 'max_disk_space') + 'enable_disk_assist', 'high_watermark', 'low_watermark', 'max_file_size', 'max_disk_space', + 'spool_directory') widgets = { 'enabled': CheckboxInput(attrs={"class": "js-switch"}), @@ -327,6 +334,7 @@ class Meta: 'low_watermark': NumberInput(attrs={'class': 'form-control'}), 'max_file_size': NumberInput(attrs={'class': 'form-control'}), 'max_disk_space': NumberInput(attrs={'class': 'form-control'}), + 'spool_directory': TextInput(attrs={'class': 'form-control'}), } def clean_index_pattern(self): @@ -355,7 +363,7 @@ class Meta: fields = ('name', 'enabled', 'db', 'collection', 'uristr', 'x509_certificate', 'send_as_raw', 'queue_size', 'dequeue_size', 'queue_timeout_shutdown', 'max_workers', 'new_worker_minimum_messages', 'worker_timeout_shutdown', 'enable_retry', 'enable_disk_assist', 'high_watermark', 'low_watermark', - 'max_file_size', 'max_disk_space') + 'max_file_size', 'max_disk_space', 'spool_directory') widgets = { 'enabled': CheckboxInput(attrs={"class": " js-switch"}), @@ -376,6 +384,7 @@ class Meta: 'low_watermark': NumberInput(attrs={'class': 'form-control'}), 'max_file_size': NumberInput(attrs={'class': 'form-control'}), 'max_disk_space': NumberInput(attrs={'class': 'form-control'}), + 'spool_directory': TextInput(attrs={'class': 'form-control'}), } @@ -387,7 +396,7 @@ class Meta: 'partitions_auto', 'confParam', 'topicConfParam', 'queue_size', 'dequeue_size', 'queue_timeout_shutdown', 'max_workers', 'new_worker_minimum_messages', 'worker_timeout_shutdown', 'enable_retry', 'enable_disk_assist', 'high_watermark', 'low_watermark', 'max_file_size', - 'max_disk_space') + 'max_disk_space', 'spool_directory') widgets = { 'enabled': CheckboxInput(attrs={"class": " js-switch"}), @@ -413,6 +422,7 @@ class Meta: 'low_watermark': NumberInput(attrs={'class': 'form-control'}), 'max_file_size': NumberInput(attrs={'class': 'form-control'}), 'max_disk_space': NumberInput(attrs={'class': 'form-control'}), + 'spool_directory': TextInput(attrs={'class': 'form-control'}), } def __init__(self, *args, **kwargs): diff --git a/vulture_os/applications/logfwd/models.py b/vulture_os/applications/logfwd/models.py index 6e1c0b40..c1d252f2 100644 --- a/vulture_os/applications/logfwd/models.py +++ b/vulture_os/applications/logfwd/models.py @@ -168,6 +168,15 @@ class LogOM (models.Model): verbose_name=_("Max disk space used by the queue in MB (set to zero to disable)"), validators=[MinValueValidator(0)] ) + spool_directory = models.TextField( + default="/var/tmp", + null=False, + help_text=_("Defines an existing folder to store queue files into"), + verbose_name=_("Existing folder to store queue files to"), + validators=[RegexValidator( + regex=r"^/.*$", + message="Value should be a valid fullpath, beginning with a '/'" + )]) send_as_raw = models.BooleanField( default=False, help_text=_("Send logs without any modification"), diff --git a/vulture_os/applications/templates/apps/logfwd_LogOMElasticSearch.html b/vulture_os/applications/templates/apps/logfwd_LogOMElasticSearch.html index e72e793a..a169abd5 100644 --- a/vulture_os/applications/templates/apps/logfwd_LogOMElasticSearch.html +++ b/vulture_os/applications/templates/apps/logfwd_LogOMElasticSearch.html @@ -319,6 +319,17 @@

Form errors

+
+
+
+ +
+ {{form.spool_directory}} + {{form.spool_directory.errors|safe}} +
+
+
+
diff --git a/vulture_os/applications/templates/apps/logfwd_LogOMFWD.html b/vulture_os/applications/templates/apps/logfwd_LogOMFWD.html index 0c92d0d1..c29485ca 100644 --- a/vulture_os/applications/templates/apps/logfwd_LogOMFWD.html +++ b/vulture_os/applications/templates/apps/logfwd_LogOMFWD.html @@ -296,6 +296,17 @@

Form errors

+
+
+
+ +
+ {{form.spool_directory}} + {{form.spool_directory.errors|safe}} +
+
+
+
diff --git a/vulture_os/applications/templates/apps/logfwd_LogOMFile.html b/vulture_os/applications/templates/apps/logfwd_LogOMFile.html index 5925a25e..6fed46bc 100644 --- a/vulture_os/applications/templates/apps/logfwd_LogOMFile.html +++ b/vulture_os/applications/templates/apps/logfwd_LogOMFile.html @@ -295,6 +295,17 @@

 {% translate "Local F +
+
+
+ +
+ {{form.spool_directory}} + {{form.spool_directory.errors|safe}} +
+
+
+
diff --git a/vulture_os/applications/templates/apps/logfwd_LogOMHIREDIS.html b/vulture_os/applications/templates/apps/logfwd_LogOMHIREDIS.html index 907e2a5d..ad4c4076 100644 --- a/vulture_os/applications/templates/apps/logfwd_LogOMHIREDIS.html +++ b/vulture_os/applications/templates/apps/logfwd_LogOMHIREDIS.html @@ -341,6 +341,17 @@

Form errors

+
+
+
+ +
+ {{form.spool_directory}} + {{form.spool_directory.errors|safe}} +
+
+
+
diff --git a/vulture_os/applications/templates/apps/logfwd_LogOMKAFKA.html b/vulture_os/applications/templates/apps/logfwd_LogOMKAFKA.html index c4dc52c7..d6b9fd19 100644 --- a/vulture_os/applications/templates/apps/logfwd_LogOMKAFKA.html +++ b/vulture_os/applications/templates/apps/logfwd_LogOMKAFKA.html @@ -319,6 +319,17 @@

Form errors

+
+
+
+ +
+ {{form.spool_directory}} + {{form.spool_directory.errors|safe}} +
+
+
+
diff --git a/vulture_os/applications/templates/apps/logfwd_LogOMMongoDB.html b/vulture_os/applications/templates/apps/logfwd_LogOMMongoDB.html index 9d91697b..4d386073 100644 --- a/vulture_os/applications/templates/apps/logfwd_LogOMMongoDB.html +++ b/vulture_os/applications/templates/apps/logfwd_LogOMMongoDB.html @@ -284,6 +284,17 @@

 {% translate "MongoDB +
+
+
+ +
+ {{form.spool_directory}} + {{form.spool_directory.errors|safe}} +
+
+
+
diff --git a/vulture_os/applications/templates/apps/logfwd_LogOMRELP.html b/vulture_os/applications/templates/apps/logfwd_LogOMRELP.html index d7165a76..7a50a143 100644 --- a/vulture_os/applications/templates/apps/logfwd_LogOMRELP.html +++ b/vulture_os/applications/templates/apps/logfwd_LogOMRELP.html @@ -275,6 +275,17 @@

Form errors

+
+
+
+ +
+ {{form.spool_directory}} + {{form.spool_directory.errors|safe}} +
+
+
+