LOGOM::ADDED:: Add the ability to define the spooling directory for D…

…A queues
VultureProject · Mar 20, 2024 · 420eb47 · 420eb47
1 parent c4709d8
commit 420eb47
Show file tree

Hide file tree

Showing 10 changed files with 106 additions and 9 deletions.
diff --git a/CHANGELOG b/CHANGELOG
@@ -9,6 +9,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 ### Added
 - [LOG_FORWARDER] [REDIS] Support for queues, sets, publish and streams modes for log insertion
 - [LOGOMELASTICSEARCH] Handle log insertion errors returned by elasticsearch
+- [LOGOM] Add the ability to define the spooling directory for DA queues
 ### Changed
 - [DEPENDENCIES] Upgrade djongo and code for pymongo>=4
 - [LOGOM] Code and logic factorization

diff --git a/vulture_os/applications/logfwd/form.py b/vulture_os/applications/logfwd/form.py
@@ -50,7 +50,8 @@ class Meta:
         model = LogOM
         fields = ('name', 'enabled', 'send_as_raw', 'queue_size', 'dequeue_size', 'queue_timeout_shutdown',
                   'max_workers', 'new_worker_minimum_messages', 'worker_timeout_shutdown', 'enable_retry',
-                  'enable_disk_assist', 'high_watermark', 'low_watermark', 'max_file_size', 'max_disk_space')
+                  'enable_disk_assist', 'high_watermark', 'low_watermark', 'max_file_size', 'max_disk_space',
+                  'spool_directory')
 
         widgets = {
             'enabled': CheckboxInput(attrs={"class": " js-switch"}),
@@ -68,11 +69,12 @@ class Meta:
             'low_watermark': NumberInput(attrs={'class': 'form-control'}),
             'max_file_size': NumberInput(attrs={'class': 'form-control'}),
             'max_disk_space': NumberInput(attrs={'class': 'form-control'}),
+            'spool_directory': TextInput(attrs={'class': 'form-control'}),
         }
 
     def __init__(self, *args, **kwargs):
         super().__init__(*args, **kwargs)
-        for field_name in ['high_watermark', 'low_watermark', 'max_file_size', 'max_disk_space']:
+        for field_name in ['high_watermark', 'low_watermark', 'max_file_size', 'max_disk_space', 'spool_directory']:
             self.fields[field_name].required = False
 
     def clean_name(self):
@@ -109,7 +111,7 @@ class Meta:
         fields = ('name', 'enabled', 'file', 'flush_interval', 'async_writing', 'send_as_raw', 'retention_time',
                   'rotation_period', 'queue_size', 'dequeue_size', 'queue_timeout_shutdown', 'max_workers',
                   'new_worker_minimum_messages', 'worker_timeout_shutdown', 'enable_retry', 'enable_disk_assist',
-                  'high_watermark', 'low_watermark', 'max_file_size', 'max_disk_space')
+                  'high_watermark', 'low_watermark', 'max_file_size', 'max_disk_space', 'spool_directory')
 
         widgets = {
             'enabled': CheckboxInput(attrs={"class": " js-switch"}),
@@ -132,6 +134,7 @@ class Meta:
             'low_watermark': NumberInput(attrs={'class': 'form-control'}),
             'max_file_size': NumberInput(attrs={'class': 'form-control'}),
             'max_disk_space': NumberInput(attrs={'class': 'form-control'}),
+            'spool_directory': TextInput(attrs={'class': 'form-control'}),
         }
 
     def clean_name(self):
@@ -154,7 +157,7 @@ class Meta:
         fields = ('name', 'enabled', 'target', 'port', 'tls_enabled', 'x509_certificate', 'send_as_raw', 'queue_size',
                   'dequeue_size', 'queue_timeout_shutdown', 'max_workers', 'new_worker_minimum_messages',
                   'worker_timeout_shutdown', 'enable_retry', 'enable_disk_assist', 'high_watermark', 'low_watermark',
-                  'max_file_size', 'max_disk_space')
+                  'max_file_size', 'max_disk_space', 'spool_directory')
 
         widgets = {
             'enabled': CheckboxInput(attrs={"class": " js-switch"}),
@@ -176,6 +179,7 @@ class Meta:
             'low_watermark': NumberInput(attrs={'class': 'form-control'}),
             'max_file_size': NumberInput(attrs={'class': 'form-control'}),
             'max_disk_space': NumberInput(attrs={'class': 'form-control'}),
+            'spool_directory': TextInput(attrs={'class': 'form-control'}),
         }
 
     def __init__(self, *args, **kwargs):
@@ -197,7 +201,7 @@ class Meta:
                   'expire_key', 'stream_outfield', 'stream_capacitylimit', 'send_as_raw',
                   'queue_size', 'dequeue_size', 'queue_timeout_shutdown', 'max_workers', 'new_worker_minimum_messages',
                   'worker_timeout_shutdown', 'enable_retry', 'enable_disk_assist', 'high_watermark', 'low_watermark',
-                  'max_file_size', 'max_disk_space')
+                  'max_file_size', 'max_disk_space', 'spool_directory')
 
         widgets = {
             'enabled': CheckboxInput(attrs={'class': 'js-switch'}),
@@ -225,6 +229,7 @@ class Meta:
             'low_watermark': NumberInput(attrs={'class': 'form-control'}),
             'max_file_size': NumberInput(attrs={'class': 'form-control'}),
             'max_disk_space': NumberInput(attrs={'class': 'form-control'}),
+            'spool_directory': TextInput(attrs={'class': 'form-control'}),
         }
 
     def clean_key(self):
@@ -250,7 +255,7 @@ class Meta:
         fields = ('name', 'enabled', 'target', 'port', 'protocol', 'zip_level', 'queue_size', 'dequeue_size',
                   'queue_timeout_shutdown', 'max_workers', 'new_worker_minimum_messages', 'worker_timeout_shutdown',
                   'enable_retry', 'enable_disk_assist', 'high_watermark', 'low_watermark', 'max_file_size',
-                  'max_disk_space', 'ratelimit_interval', 'ratelimit_burst', 'send_as_raw')
+                  'max_disk_space', 'spool_directory', 'ratelimit_interval', 'ratelimit_burst', 'send_as_raw')
 
         widgets = {
             'enabled': CheckboxInput(attrs={"class": " js-switch"}),
@@ -271,6 +276,7 @@ class Meta:
             'low_watermark': NumberInput(attrs={'class': 'form-control'}),
             'max_file_size': NumberInput(attrs={'class': 'form-control'}),
             'max_disk_space': NumberInput(attrs={'class': 'form-control'}),
+            'spool_directory': TextInput(attrs={'class': 'form-control'}),
             'ratelimit_interval': NumberInput(attrs={'class': 'form-control'}),
             'ratelimit_burst': NumberInput(attrs={'class': 'form-control'}),
             'send_as_raw': CheckboxInput(attrs={'class': 'form-control js-switch'})
@@ -301,7 +307,8 @@ class Meta:
         fields = ('name', 'enabled', 'servers', 'es8_compatibility', 'data_stream_mode', 'retry_on_els_failures', 'index_pattern', 'uid', 'pwd',
                   'x509_certificate', 'send_as_raw', 'queue_size', 'dequeue_size', 'queue_timeout_shutdown',
                   'max_workers', 'new_worker_minimum_messages', 'worker_timeout_shutdown', 'enable_retry',
-                  'enable_disk_assist', 'high_watermark', 'low_watermark', 'max_file_size', 'max_disk_space')
+                  'enable_disk_assist', 'high_watermark', 'low_watermark', 'max_file_size', 'max_disk_space',
+                  'spool_directory')
 
         widgets = {
             'enabled': CheckboxInput(attrs={"class": "js-switch"}),
@@ -327,6 +334,7 @@ class Meta:
             'low_watermark': NumberInput(attrs={'class': 'form-control'}),
             'max_file_size': NumberInput(attrs={'class': 'form-control'}),
             'max_disk_space': NumberInput(attrs={'class': 'form-control'}),
+            'spool_directory': TextInput(attrs={'class': 'form-control'}),
         }
 
     def clean_index_pattern(self):
@@ -355,7 +363,7 @@ class Meta:
         fields = ('name', 'enabled', 'db', 'collection', 'uristr', 'x509_certificate', 'send_as_raw', 'queue_size',
                   'dequeue_size', 'queue_timeout_shutdown', 'max_workers', 'new_worker_minimum_messages',
                   'worker_timeout_shutdown', 'enable_retry', 'enable_disk_assist', 'high_watermark', 'low_watermark',
-                  'max_file_size', 'max_disk_space')
+                  'max_file_size', 'max_disk_space', 'spool_directory')
 
         widgets = {
             'enabled': CheckboxInput(attrs={"class": " js-switch"}),
@@ -376,6 +384,7 @@ class Meta:
             'low_watermark': NumberInput(attrs={'class': 'form-control'}),
             'max_file_size': NumberInput(attrs={'class': 'form-control'}),
             'max_disk_space': NumberInput(attrs={'class': 'form-control'}),
+            'spool_directory': TextInput(attrs={'class': 'form-control'}),
         }
 
 
@@ -387,7 +396,7 @@ class Meta:
                   'partitions_auto', 'confParam', 'topicConfParam', 'queue_size', 'dequeue_size',
                   'queue_timeout_shutdown', 'max_workers', 'new_worker_minimum_messages', 'worker_timeout_shutdown',
                   'enable_retry', 'enable_disk_assist', 'high_watermark', 'low_watermark', 'max_file_size',
-                  'max_disk_space')
+                  'max_disk_space', 'spool_directory')
 
         widgets = {
             'enabled': CheckboxInput(attrs={"class": " js-switch"}),
@@ -413,6 +422,7 @@ class Meta:
             'low_watermark': NumberInput(attrs={'class': 'form-control'}),
             'max_file_size': NumberInput(attrs={'class': 'form-control'}),
             'max_disk_space': NumberInput(attrs={'class': 'form-control'}),
+            'spool_directory': TextInput(attrs={'class': 'form-control'}),
         }
 
     def __init__(self, *args, **kwargs):

diff --git a/vulture_os/applications/logfwd/models.py b/vulture_os/applications/logfwd/models.py
@@ -168,6 +168,15 @@ class LogOM (models.Model):
         verbose_name=_("Max disk space used by the queue in MB (set to zero to disable)"),
         validators=[MinValueValidator(0)]
     )
+    spool_directory = models.TextField(
+        default="/var/tmp",
+        null=False,
+        help_text=_("Defines an existing folder to store queue files into"),
+        verbose_name=_("Existing folder to store queue files to"),
+        validators=[RegexValidator(
+                regex=r"^/.*$",
+                message="Value should be a valid fullpath, beginning with a '/'"
+        )])
     send_as_raw = models.BooleanField(
         default=False,
         help_text=_("Send logs without any modification"),

diff --git a/vulture_os/applications/templates/apps/logfwd_LogOMElasticSearch.html b/vulture_os/applications/templates/apps/logfwd_LogOMElasticSearch.html
@@ -319,6 +319,17 @@ <h4><i class="icon fa fa-ban"></i> Form errors </h4>
                         </div>
                       </div>
                     </div>
+                    <div class="row">
+                      <div class="col-md-12">
+                        <div class="form-group">
+                          <label class="col-sm-4 control-label">{{form.spool_directory.label}}</label>
+                          <div class="col-sm-5">
+                            {{form.spool_directory}}
+                            {{form.spool_directory.errors|safe}}
+                          </div>
+                        </div>
+                      </div>
+                    </div>
                   </div> <!-- /.tab-disk -->
                 </div> <!-- /.tab-queue -->
               </div> <!-- /.advanced_options -->

diff --git a/vulture_os/applications/templates/apps/logfwd_LogOMFWD.html b/vulture_os/applications/templates/apps/logfwd_LogOMFWD.html
@@ -296,6 +296,17 @@ <h4><i class="icon fa fa-ban"></i> Form errors </h4>
                         </div>
                       </div>
                     </div>
+                    <div class="row">
+                      <div class="col-md-12">
+                        <div class="form-group">
+                          <label class="col-sm-4 control-label">{{form.spool_directory.label}}</label>
+                          <div class="col-sm-5">
+                            {{form.spool_directory}}
+                            {{form.spool_directory.errors|safe}}
+                          </div>
+                        </div>
+                      </div>
+                    </div>
                   </div> <!-- /.tab-disk -->
                 </div> <!-- /.tab-queue -->
               </div> <!-- Advanced tab -->

diff --git a/vulture_os/applications/templates/apps/logfwd_LogOMFile.html b/vulture_os/applications/templates/apps/logfwd_LogOMFile.html
@@ -295,6 +295,17 @@ <h1 class="panel-title"><i class="fa fa-sitemap">&nbsp;</i>{% translate "Local F
                         </div>
                       </div>
                     </div>
+                    <div class="row">
+                      <div class="col-md-12">
+                        <div class="form-group">
+                          <label class="col-sm-4 control-label">{{form.spool_directory.label}}</label>
+                          <div class="col-sm-5">
+                            {{form.spool_directory}}
+                            {{form.spool_directory.errors|safe}}
+                          </div>
+                        </div>
+                      </div>
+                    </div>
                   </div> <!-- /.tab-disk -->
                 </div> <!-- /.tab-queue -->
               </div> <!-- /.advanced_options -->

diff --git a/vulture_os/applications/templates/apps/logfwd_LogOMHIREDIS.html b/vulture_os/applications/templates/apps/logfwd_LogOMHIREDIS.html
@@ -341,6 +341,17 @@ <h4><i class="icon fa fa-ban"></i> Form errors </h4>
                         </div>
                       </div>
                     </div>
+                    <div class="row">
+                      <div class="col-md-12">
+                        <div class="form-group">
+                          <label class="col-sm-4 control-label">{{form.spool_directory.label}}</label>
+                          <div class="col-sm-5">
+                            {{form.spool_directory}}
+                            {{form.spool_directory.errors|safe}}
+                          </div>
+                        </div>
+                      </div>
+                    </div>
                   </div> <!-- /.tab-disk -->
                 </div> <!-- /.tab-queue -->
               </div> <!-- /.advanced_options -->

diff --git a/vulture_os/applications/templates/apps/logfwd_LogOMKAFKA.html b/vulture_os/applications/templates/apps/logfwd_LogOMKAFKA.html
@@ -319,6 +319,17 @@ <h4><i class="icon fa fa-ban"></i> Form errors </h4>
                         </div>
                       </div>
                     </div>
+                    <div class="row">
+                      <div class="col-md-12">
+                        <div class="form-group">
+                          <label class="col-sm-4 control-label">{{form.spool_directory.label}}</label>
+                          <div class="col-sm-5">
+                            {{form.spool_directory}}
+                            {{form.spool_directory.errors|safe}}
+                          </div>
+                        </div>
+                      </div>
+                    </div>
                   </div> <!-- /.tab-disk -->
                 </div> <!-- /.tab-queue -->
               </div> <!-- /.advanced_options -->

diff --git a/vulture_os/applications/templates/apps/logfwd_LogOMMongoDB.html b/vulture_os/applications/templates/apps/logfwd_LogOMMongoDB.html
@@ -284,6 +284,17 @@ <h1 class="panel-title"><i class="fa fa-sitemap">&nbsp;</i>{% translate "MongoDB
                         </div>
                       </div>
                     </div>
+                    <div class="row">
+                      <div class="col-md-12">
+                        <div class="form-group">
+                          <label class="col-sm-4 control-label">{{form.spool_directory.label}}</label>
+                          <div class="col-sm-5">
+                            {{form.spool_directory}}
+                            {{form.spool_directory.errors|safe}}
+                          </div>
+                        </div>
+                      </div>
+                    </div>
                   </div> <!-- /.tab-disk -->
                 </div> <!-- /.tab-queue -->
               </div> <!-- /.advanced_options -->

diff --git a/vulture_os/applications/templates/apps/logfwd_LogOMRELP.html b/vulture_os/applications/templates/apps/logfwd_LogOMRELP.html
@@ -275,6 +275,17 @@ <h4><i class="icon fa fa-ban"></i> Form errors </h4>
                         </div>
                       </div>
                     </div>
+                    <div class="row">
+                      <div class="col-md-12">
+                        <div class="form-group">
+                          <label class="col-sm-4 control-label">{{form.spool_directory.label}}</label>
+                          <div class="col-sm-5">
+                            {{form.spool_directory}}
+                            {{form.spool_directory.errors|safe}}
+                          </div>
+                        </div>
+                      </div>
+                    </div>
                   </div> <!-- /.tab-disk -->
                 </div> <!-- /.tab-queue -->
               </div> <!-- /.advanced_options -->