[LOGOM] various improvements (#411)

### Added - [LOGOM] Add the ability to define the spooling directory for DA queues ### Changed - [LOGOM] Code and logic factorization - [LOGOM] Allow to deactivate limitation of the size of DA queues on disk - [LOGOM] [ELASTICSEARCH] Limit available certificates to non-CA ones - [LOGOM] Increment checkpoint interval for DA queues, to increase performances --------- Co-authored-by: Fabien Amelinck <[email protected]>
VultureProject · Sep 2, 2024 · b23f32f · b23f32f
1 parent 7bc3d4f
commit b23f32f
Show file tree

Hide file tree

Showing 19 changed files with 380 additions and 436 deletions.
diff --git a/vulture_os/applications/logfwd/config/om_elasticsearch.tpl b/vulture_os/applications/logfwd/config/om_elasticsearch.tpl
@@ -48,11 +48,11 @@
         {%- if enable_disk_assist %}
             queue.highWatermark="{{high_watermark}}"
             queue.lowWatermark="{{low_watermark}}"
-            queue.spoolDirectory="/var/tmp"
+            queue.spoolDirectory="{{spool_directory}}"
             queue.filename="{{output_name}}_disk-queue"
             queue.maxFileSize="{{max_file_size}}m"
             queue.maxDiskSpace="{{max_disk_space}}m"
-            queue.checkpointInterval="128"
+            queue.checkpointInterval="1024"
             queue.saveOnShutdown="on"
         {%- endif -%} {# if enable_disk_assist #}
     {%- endif -%} {# if enable_retry #}

diff --git a/vulture_os/applications/logfwd/config/om_elasticsearch_pre.tpl b/vulture_os/applications/logfwd/config/om_elasticsearch_pre.tpl
@@ -69,11 +69,11 @@ ruleset(name="{{output_name}}_retry") {
         {%- if enable_disk_assist %}
             queue.highWatermark="{{high_watermark}}"
             queue.lowWatermark="{{low_watermark}}"
-            queue.spoolDirectory="/var/tmp"
+            queue.spoolDirectory="{{spool_directory}}"
             queue.filename="{{output_name}}_retry_disk-queue"
             queue.maxFileSize="{{max_file_size}}m"
             queue.maxDiskSpace="{{max_disk_space}}m"
-            queue.checkpointInterval="128"
+            queue.checkpointInterval="1024"
             queue.saveOnShutdown="on"
         {%- endif -%} {# if enable_disk_assist #}
     {%- endif -%} {# if enable_retry #}

diff --git a/vulture_os/applications/logfwd/config/om_file.tpl b/vulture_os/applications/logfwd/config/om_file.tpl
@@ -28,11 +28,11 @@
             {%- if enable_disk_assist %}
             queue.highWatermark="{{high_watermark}}"
             queue.lowWatermark="{{low_watermark}}"
-            queue.spoolDirectory="/var/tmp"
+            queue.spoolDirectory="{{spool_directory}}"
             queue.filename="{{output_name}}_disk-queue"
             queue.maxFileSize="{{max_file_size}}m"
             queue.maxDiskSpace="{{max_disk_space}}m"
-            queue.checkpointInterval="128"
+            queue.checkpointInterval="1024"
             queue.saveOnShutdown="on"
             {%- endif -%} {# if enable_disk_assist #}
             {%- endif -%} {# if enable_retry #}

diff --git a/vulture_os/applications/logfwd/config/om_fwd.tpl b/vulture_os/applications/logfwd/config/om_fwd.tpl
@@ -31,11 +31,11 @@
             {%- if enable_disk_assist %}
             queue.highWatermark="{{high_watermark}}"
             queue.lowWatermark="{{low_watermark}}"
-            queue.spoolDirectory="/var/tmp"
+            queue.spoolDirectory="{{spool_directory}}"
             queue.filename="{{output_name}}_disk-queue"
             queue.maxFileSize="{{max_file_size}}m"
             queue.maxDiskSpace="{{max_disk_space}}m"
-            queue.checkpointInterval="128"
+            queue.checkpointInterval="1024"
             queue.saveOnShutdown="on"
             {%- endif -%} {# if enable_disk_assist #}
             {%- endif -%} {# if enable_retry #}

diff --git a/vulture_os/applications/logfwd/config/om_hiredis.tpl b/vulture_os/applications/logfwd/config/om_hiredis.tpl
@@ -43,11 +43,11 @@
         {%- if enable_disk_assist %}
             queue.highWatermark="{{high_watermark}}"
             queue.lowWatermark="{{low_watermark}}"
-            queue.spoolDirectory="/var/tmp"
+            queue.spoolDirectory="{{spool_directory}}"
             queue.filename="{{output_name}}_disk-queue"
             queue.maxFileSize="{{max_file_size}}m"
             queue.maxDiskSpace="{{max_disk_space}}m"
-            queue.checkpointInterval="128"
+            queue.checkpointInterval="1024"
             queue.saveOnShutdown="on"
         {%- endif -%} {# if enable_disk_assist #}
     {%- endif -%} {# if enable_retry #}

diff --git a/vulture_os/applications/logfwd/config/om_kafka.tpl b/vulture_os/applications/logfwd/config/om_kafka.tpl
@@ -48,11 +48,11 @@
         {%- if enable_disk_assist %}
             queue.highWatermark="{{high_watermark}}"
             queue.lowWatermark="{{low_watermark}}"
-            queue.spoolDirectory="/var/tmp"
+            queue.spoolDirectory="{{spool_directory}}"
             queue.filename="{{output_name}}_disk-queue"
             queue.maxFileSize="{{max_file_size}}m"
             queue.maxDiskSpace="{{max_disk_space}}m"
-            queue.checkpointInterval="128"
+            queue.checkpointInterval="1024"
             queue.saveOnShutdown='"on"'
         {%- endif -%} {# if enable_disk_assist #}
     {%- endif -%} {# if enable_retry #}

diff --git a/vulture_os/applications/logfwd/config/om_mongodb.tpl b/vulture_os/applications/logfwd/config/om_mongodb.tpl
@@ -30,11 +30,11 @@
             {%- if enable_disk_assist %}
             queue.highWatermark="{{high_watermark}}"
             queue.lowWatermark="{{low_watermark}}"
-            queue.spoolDirectory="/var/tmp"
+            queue.spoolDirectory="{{spool_directory}}"
             queue.filename="{{output_name}}_disk-queue"
             queue.maxFileSize="{{max_file_size}}m"
             queue.maxDiskSpace="{{max_disk_space}}m"
-            queue.checkpointInterval="128"
+            queue.checkpointInterval="1024"
             queue.saveOnShutdown="on"
             {%- endif -%} {# if enable_disk_assist #}
             {%- endif -%} {# if enable_retry #}

diff --git a/vulture_os/applications/logfwd/config/om_relp.tpl b/vulture_os/applications/logfwd/config/om_relp.tpl
@@ -27,11 +27,11 @@
             {%- if enable_disk_assist %}
             queue.highWatermark="{{high_watermark}}"
             queue.lowWatermark="{{low_watermark}}"
-            queue.spoolDirectory="/var/tmp"
+            queue.spoolDirectory="{{spool_directory}}"
             queue.filename="{{output_name}}_disk-queue"
             queue.maxFileSize="{{max_file_size}}m"
             queue.maxDiskSpace="{{max_disk_space}}m"
-            queue.checkpointInterval="128"
+            queue.checkpointInterval="1024"
             queue.saveOnShutdown="on"
             {%- endif -%} {# if enable_disk_assist #}
             {%- endif -%} {# if enable_retry #}