diff --git a/vulture_os/applications/logfwd/config/om_elasticsearch.tpl b/vulture_os/applications/logfwd/config/om_elasticsearch.tpl index 491cfeff..553f20a1 100644 --- a/vulture_os/applications/logfwd/config/om_elasticsearch.tpl +++ b/vulture_os/applications/logfwd/config/om_elasticsearch.tpl @@ -48,11 +48,11 @@ {%- if enable_disk_assist %} queue.highWatermark="{{high_watermark}}" queue.lowWatermark="{{low_watermark}}" - queue.spoolDirectory="/var/tmp" + queue.spoolDirectory="{{spool_directory}}" queue.filename="{{output_name}}_disk-queue" queue.maxFileSize="{{max_file_size}}m" queue.maxDiskSpace="{{max_disk_space}}m" - queue.checkpointInterval="128" + queue.checkpointInterval="1024" queue.saveOnShutdown="on" {%- endif -%} {# if enable_disk_assist #} {%- endif -%} {# if enable_retry #} diff --git a/vulture_os/applications/logfwd/config/om_elasticsearch_pre.tpl b/vulture_os/applications/logfwd/config/om_elasticsearch_pre.tpl index ccdb7105..9ff54ce5 100644 --- a/vulture_os/applications/logfwd/config/om_elasticsearch_pre.tpl +++ b/vulture_os/applications/logfwd/config/om_elasticsearch_pre.tpl @@ -69,11 +69,11 @@ ruleset(name="{{output_name}}_retry") { {%- if enable_disk_assist %} queue.highWatermark="{{high_watermark}}" queue.lowWatermark="{{low_watermark}}" - queue.spoolDirectory="/var/tmp" + queue.spoolDirectory="{{spool_directory}}" queue.filename="{{output_name}}_retry_disk-queue" queue.maxFileSize="{{max_file_size}}m" queue.maxDiskSpace="{{max_disk_space}}m" - queue.checkpointInterval="128" + queue.checkpointInterval="1024" queue.saveOnShutdown="on" {%- endif -%} {# if enable_disk_assist #} {%- endif -%} {# if enable_retry #} diff --git a/vulture_os/applications/logfwd/config/om_file.tpl b/vulture_os/applications/logfwd/config/om_file.tpl index 172fa38e..6000fa7c 100644 --- a/vulture_os/applications/logfwd/config/om_file.tpl +++ b/vulture_os/applications/logfwd/config/om_file.tpl @@ -28,11 +28,11 @@ {%- if enable_disk_assist %} queue.highWatermark="{{high_watermark}}" queue.lowWatermark="{{low_watermark}}" - queue.spoolDirectory="/var/tmp" + queue.spoolDirectory="{{spool_directory}}" queue.filename="{{output_name}}_disk-queue" queue.maxFileSize="{{max_file_size}}m" queue.maxDiskSpace="{{max_disk_space}}m" - queue.checkpointInterval="128" + queue.checkpointInterval="1024" queue.saveOnShutdown="on" {%- endif -%} {# if enable_disk_assist #} {%- endif -%} {# if enable_retry #} diff --git a/vulture_os/applications/logfwd/config/om_fwd.tpl b/vulture_os/applications/logfwd/config/om_fwd.tpl index ca87585a..8f5a2eb4 100644 --- a/vulture_os/applications/logfwd/config/om_fwd.tpl +++ b/vulture_os/applications/logfwd/config/om_fwd.tpl @@ -31,11 +31,11 @@ {%- if enable_disk_assist %} queue.highWatermark="{{high_watermark}}" queue.lowWatermark="{{low_watermark}}" - queue.spoolDirectory="/var/tmp" + queue.spoolDirectory="{{spool_directory}}" queue.filename="{{output_name}}_disk-queue" queue.maxFileSize="{{max_file_size}}m" queue.maxDiskSpace="{{max_disk_space}}m" - queue.checkpointInterval="128" + queue.checkpointInterval="1024" queue.saveOnShutdown="on" {%- endif -%} {# if enable_disk_assist #} {%- endif -%} {# if enable_retry #} diff --git a/vulture_os/applications/logfwd/config/om_hiredis.tpl b/vulture_os/applications/logfwd/config/om_hiredis.tpl index 451dcea2..16fca1db 100644 --- a/vulture_os/applications/logfwd/config/om_hiredis.tpl +++ b/vulture_os/applications/logfwd/config/om_hiredis.tpl @@ -43,11 +43,11 @@ {%- if enable_disk_assist %} queue.highWatermark="{{high_watermark}}" queue.lowWatermark="{{low_watermark}}" - queue.spoolDirectory="/var/tmp" + queue.spoolDirectory="{{spool_directory}}" queue.filename="{{output_name}}_disk-queue" queue.maxFileSize="{{max_file_size}}m" queue.maxDiskSpace="{{max_disk_space}}m" - queue.checkpointInterval="128" + queue.checkpointInterval="1024" queue.saveOnShutdown="on" {%- endif -%} {# if enable_disk_assist #} {%- endif -%} {# if enable_retry #} diff --git a/vulture_os/applications/logfwd/config/om_kafka.tpl b/vulture_os/applications/logfwd/config/om_kafka.tpl index 33e3064e..4cccf398 100644 --- a/vulture_os/applications/logfwd/config/om_kafka.tpl +++ b/vulture_os/applications/logfwd/config/om_kafka.tpl @@ -48,11 +48,11 @@ {%- if enable_disk_assist %} queue.highWatermark="{{high_watermark}}" queue.lowWatermark="{{low_watermark}}" - queue.spoolDirectory="/var/tmp" + queue.spoolDirectory="{{spool_directory}}" queue.filename="{{output_name}}_disk-queue" queue.maxFileSize="{{max_file_size}}m" queue.maxDiskSpace="{{max_disk_space}}m" - queue.checkpointInterval="128" + queue.checkpointInterval="1024" queue.saveOnShutdown='"on"' {%- endif -%} {# if enable_disk_assist #} {%- endif -%} {# if enable_retry #} diff --git a/vulture_os/applications/logfwd/config/om_mongodb.tpl b/vulture_os/applications/logfwd/config/om_mongodb.tpl index 03758497..8d979ced 100644 --- a/vulture_os/applications/logfwd/config/om_mongodb.tpl +++ b/vulture_os/applications/logfwd/config/om_mongodb.tpl @@ -30,11 +30,11 @@ {%- if enable_disk_assist %} queue.highWatermark="{{high_watermark}}" queue.lowWatermark="{{low_watermark}}" - queue.spoolDirectory="/var/tmp" + queue.spoolDirectory="{{spool_directory}}" queue.filename="{{output_name}}_disk-queue" queue.maxFileSize="{{max_file_size}}m" queue.maxDiskSpace="{{max_disk_space}}m" - queue.checkpointInterval="128" + queue.checkpointInterval="1024" queue.saveOnShutdown="on" {%- endif -%} {# if enable_disk_assist #} {%- endif -%} {# if enable_retry #} diff --git a/vulture_os/applications/logfwd/config/om_relp.tpl b/vulture_os/applications/logfwd/config/om_relp.tpl index 3646ca5a..1f78b776 100644 --- a/vulture_os/applications/logfwd/config/om_relp.tpl +++ b/vulture_os/applications/logfwd/config/om_relp.tpl @@ -27,11 +27,11 @@ {%- if enable_disk_assist %} queue.highWatermark="{{high_watermark}}" queue.lowWatermark="{{low_watermark}}" - queue.spoolDirectory="/var/tmp" + queue.spoolDirectory="{{spool_directory}}" queue.filename="{{output_name}}_disk-queue" queue.maxFileSize="{{max_file_size}}m" queue.maxDiskSpace="{{max_disk_space}}m" - queue.checkpointInterval="128" + queue.checkpointInterval="1024" queue.saveOnShutdown="on" {%- endif -%} {# if enable_disk_assist #} {%- endif -%} {# if enable_retry #} diff --git a/vulture_os/applications/logfwd/form.py b/vulture_os/applications/logfwd/form.py index fb096d64..aed68822 100644 --- a/vulture_os/applications/logfwd/form.py +++ b/vulture_os/applications/logfwd/form.py @@ -28,8 +28,8 @@ from django.forms import ModelChoiceField, ModelForm, TextInput, CheckboxInput, NumberInput, Select # Django project imports -from applications.logfwd.models import (LogOMFile, LogOMRELP, LogOMHIREDIS, LogOMFWD, LogOMElasticSearch, LogOMMongoDB, LogOMKAFKA, - OMFWD_PROTOCOL, OMHIREDIS_MODE_CHOICES) +from applications.logfwd.models import (LogOM, LogOMFile, LogOMRELP, LogOMHIREDIS, LogOMFWD, LogOMElasticSearch, + LogOMMongoDB, LogOMKAFKA, OMFWD_PROTOCOL, OMHIREDIS_MODE_CHOICES) from system.pki.models import X509Certificate # Required exceptions imports @@ -44,175 +44,128 @@ logger = logging.getLogger('gui') -class LogOMFileForm(ModelForm): +class LogOMForm(ModelForm): class Meta: - model = LogOMFile - fields = ('name', 'enabled', 'file', 'flush_interval', 'async_writing', 'send_as_raw', 'retention_time', - 'rotation_period', 'queue_size', 'dequeue_size', 'queue_timeout_shutdown', 'max_workers', - 'new_worker_minimum_messages', 'worker_timeout_shutdown', 'enable_retry', 'enable_disk_assist', - 'high_watermark', 'low_watermark', 'max_file_size', 'max_disk_space') + model = LogOM + fields = ('name', 'enabled', 'send_as_raw', 'queue_size', 'dequeue_size', 'queue_timeout_shutdown', + 'max_workers', 'new_worker_minimum_messages', 'worker_timeout_shutdown', 'enable_retry', + 'enable_disk_assist', 'high_watermark', 'low_watermark', 'max_file_size', 'max_disk_space', + 'spool_directory') widgets = { - 'enabled': CheckboxInput(attrs={"class": " js-switch"}), + 'enabled': CheckboxInput(attrs={'class': 'js-switch'}), 'name': TextInput(attrs={'class': 'form-control'}), - 'file': TextInput(attrs={'class': 'form-control'}), - 'flush_interval': NumberInput(attrs={'class': 'form-control'}), - 'async_writing': CheckboxInput(attrs={"class": " js-switch"}), - 'send_as_raw': CheckboxInput(attrs={"class": " js-switch"}), - 'retention_time': NumberInput(attrs={"class": "form-control"}), - 'rotation_period': Select(attrs={"class": "select2"}), + 'send_as_raw': CheckboxInput(attrs={'class': 'js-switch'}), 'queue_size': NumberInput(attrs={'class': 'form-control'}), 'dequeue_size': NumberInput(attrs={'class': 'form-control'}), 'queue_timeout_shutdown': NumberInput(attrs={'class': 'form-control', 'placeholder': 10}), 'max_workers': NumberInput(attrs={'class': 'form-control', 'placeholder': 1}), 'new_worker_minimum_messages': NumberInput(attrs={'class': 'form-control', 'placeholder': 'queue size / max workers'}), 'worker_timeout_shutdown': NumberInput(attrs={'class': 'form-control', 'placeholder': 60_000}), - 'enable_retry': CheckboxInput(attrs={"class": " js-switch"}), - 'enable_disk_assist': CheckboxInput(attrs={"class": " js-switch"}), + 'enable_retry': CheckboxInput(attrs={'class': 'js-switch'}), + 'enable_disk_assist': CheckboxInput(attrs={'class': 'js-switch'}), 'high_watermark': NumberInput(attrs={'class': 'form-control'}), 'low_watermark': NumberInput(attrs={'class': 'form-control'}), 'max_file_size': NumberInput(attrs={'class': 'form-control'}), - 'max_disk_space': NumberInput(attrs={'class': 'form-control'}) + 'max_disk_space': NumberInput(attrs={'class': 'form-control'}), + 'spool_directory': TextInput(attrs={'class': 'form-control'}), } def __init__(self, *args, **kwargs): super().__init__(*args, **kwargs) - self.fields['retention_time'].empty_label = None - self.fields['rotation_period'].empty_label = None - for field_name in ['high_watermark', 'low_watermark', 'max_file_size', 'max_disk_space']: + for field_name in ['high_watermark', 'low_watermark', 'max_file_size', 'max_disk_space', 'spool_directory']: self.fields[field_name].required = False def clean_name(self): - field = self.cleaned_data.get('name') - if not field: - raise ValidationError("This field is required.") - return field.replace(' ', '_') + return self.cleaned_data['name'].replace(' ', '_') - def clean_file(self): - value = self.cleaned_data.get('file') - if not value.startswith('/'): - raise ValidationError("That field needs absolute path.") - return value + def clean_spool_directory(self): + return "/" + self.cleaned_data['spool_directory'].strip("/") def clean(self): """ Verify needed fields - depending on mode chosen """ cleaned_data = super().clean() - if cleaned_data.get('enable_disk_assist') == True: - try: - assert cleaned_data.get('high_watermark'), "high_watermark" - assert cleaned_data.get('low_watermark'), "low_watermark" - except AssertionError as e: - self.add_error(str(e), "This field is required.") - else: - if cleaned_data.get('queue_size') and (cleaned_data.get('queue_size') < cleaned_data.get('high_watermark') or cleaned_data.get('queue_size') < cleaned_data.get('low_watermark')): - self.add_error("queue_size", "Queue size is lower than a watermark") - if cleaned_data.get('high_watermark') < cleaned_data.get('low_watermark'): + logger.info(self.initial) + if cleaned_data['enable_disk_assist'] == True: + if cleaned_data['queue_size'] is not None and cleaned_data['high_watermark'] is not None: + if cleaned_data['queue_size'] < cleaned_data['high_watermark']: + self.add_error("queue_size", "Queue size is lower than the high watermark") + if cleaned_data['queue_size'] is not None and cleaned_data['low_watermark'] is not None: + if cleaned_data['queue_size'] < cleaned_data['low_watermark']: + self.add_error("queue_size", "Queue size is lower than the low watermark") + if cleaned_data['low_watermark'] is not None and cleaned_data['high_watermark'] is not None: + if cleaned_data['high_watermark'] < cleaned_data['low_watermark']: self.add_error("high_watermark", "High watermark is lower than the low watermark value") self.add_error("low_watermark", "Low watermark is higher than the high watermark value") - try: - assert cleaned_data.get('max_file_size'), "max_file_size" - assert cleaned_data.get('max_disk_space'), "max_disk_space" - except AssertionError as e: - self.add_error(str(e), "This field is required.") - else: - if cleaned_data.get('max_file_size') > cleaned_data.get('max_disk_space'): + if cleaned_data['max_disk_space'] is not None and cleaned_data['max_file_size'] is not None: + if cleaned_data['max_disk_space'] > 0 and cleaned_data['max_file_size'] > cleaned_data.get('max_disk_space'): self.add_error("max_file_size", "File size is higher than the disk space") - if cleaned_data.get('new_worker_minimum_messages'): - if cleaned_data['new_worker_minimum_messages'] > cleaned_data.get('queue_size', 10000): + if cleaned_data['new_worker_minimum_messages'] is not None and cleaned_data['queue_size'] is not None: + if cleaned_data['new_worker_minimum_messages'] > cleaned_data['queue_size']: self.add_error("new_worker_minimum_messages", "This value cannot be over the queue size") return cleaned_data -class LogOMRELPForm(ModelForm): +class LogOMFileForm(LogOMForm): - class Meta: + class Meta(LogOMForm.Meta): + model = LogOMFile + fields = LogOMForm.Meta.fields + ('file', 'flush_interval', 'async_writing', 'retention_time', + 'rotation_period') + + widgets = { + 'file': TextInput(attrs={'class': 'form-control'}), + 'flush_interval': NumberInput(attrs={'class': 'form-control'}), + 'async_writing': CheckboxInput(attrs={'class': 'js-switch'}), + 'retention_time': NumberInput(attrs={'class': 'form-control'}), + 'rotation_period': Select(attrs={'class': 'select2'}), + } + widgets.update(LogOMForm.Meta.widgets) + + def clean_file(self): + value = self.cleaned_data['file'] + if not value.startswith('/'): + raise ValidationError("That field needs absolute path.") + return value + + +class LogOMRELPForm(LogOMForm): + + class Meta(LogOMForm.Meta): model = LogOMRELP - fields = ('name', 'enabled', 'target', 'port', 'tls_enabled', 'x509_certificate', 'send_as_raw', 'queue_size', - 'dequeue_size', 'queue_timeout_shutdown', 'max_workers', 'new_worker_minimum_messages', - 'worker_timeout_shutdown', 'enable_retry', 'enable_disk_assist', 'high_watermark', 'low_watermark', - 'max_file_size', 'max_disk_space') + fields = LogOMForm.Meta.fields + ('target', 'port', 'tls_enabled', 'x509_certificate') widgets = { - 'enabled': CheckboxInput(attrs={"class": " js-switch"}), - 'name': TextInput(attrs={'class': 'form-control'}), 'target': TextInput(attrs={'class': 'form-control'}), 'port': NumberInput(attrs={'class': 'form-control'}), - 'tls_enabled': CheckboxInput(attrs={"class": " js-switch"}), - 'x509_certificate': Select(attrs={'class': 'form-control select2'}), - 'send_as_raw': CheckboxInput(attrs={'class': 'form-control js-switch'}), - 'queue_size': NumberInput(attrs={'class': 'form-control'}), - 'dequeue_size': NumberInput(attrs={'class': 'form-control'}), - 'queue_timeout_shutdown': NumberInput(attrs={'class': 'form-control', 'placeholder': 10}), - 'max_workers': NumberInput(attrs={'class': 'form-control', 'placeholder': 1}), - 'new_worker_minimum_messages': NumberInput(attrs={'class': 'form-control', 'placeholder': 'queue size / max workers'}), - 'worker_timeout_shutdown': NumberInput(attrs={'class': 'form-control', 'placeholder': 60_000}), - 'enable_retry': CheckboxInput(attrs={"class": " js-switch"}), - 'enable_disk_assist': CheckboxInput(attrs={"class": " js-switch"}), - 'high_watermark': NumberInput(attrs={'class': 'form-control'}), - 'low_watermark': NumberInput(attrs={'class': 'form-control'}), - 'max_file_size': NumberInput(attrs={'class': 'form-control'}), - 'max_disk_space': NumberInput(attrs={'class': 'form-control'}) + 'tls_enabled': CheckboxInput(attrs={'class': 'js-switch'}), + 'x509_certificate': Select(attrs={'class': 'select2'}), } + widgets.update(LogOMForm.Meta.widgets) def __init__(self, *args, **kwargs): super().__init__(*args, **kwargs) self.fields['x509_certificate'].empty_label = "No TLS certificate" - for field_name in ['x509_certificate', 'high_watermark', 'low_watermark', 'max_file_size', 'max_disk_space']: - self.fields[field_name].required = False - - def clean_name(self): - field = self.cleaned_data.get('name') - if not field: - raise ValidationError("This field is required.") - return field.replace(' ', '_') def clean(self): cleaned_data = super().clean() if not cleaned_data.get('tls_enabled') and cleaned_data.get('x509_certificate'): self.add_error('tls_enabled', "You must enable tls to specify a certificate.") - if cleaned_data.get('enable_disk_assist') == True: - try: - assert cleaned_data.get('high_watermark'), "high_watermark" - assert cleaned_data.get('low_watermark'), "low_watermark" - except AssertionError as e: - self.add_error(str(e), "This field is required.") - else: - if cleaned_data.get('queue_size') and (cleaned_data.get('queue_size') < cleaned_data.get('high_watermark') or cleaned_data.get('queue_size') < cleaned_data.get('low_watermark')): - self.add_error("queue_size", "Queue size is lower than a watermark") - if cleaned_data.get('high_watermark') < cleaned_data.get('low_watermark'): - self.add_error("high_watermark", "High watermark is lower than the low watermark value") - self.add_error("low_watermark", "Low watermark is higher than the high watermark value") - try: - assert cleaned_data.get('max_file_size'), "max_file_size" - assert cleaned_data.get('max_disk_space'), "max_disk_space" - except AssertionError as e: - self.add_error(str(e), "This field is required.") - else: - if cleaned_data.get('max_file_size') > cleaned_data.get('max_disk_space'): - self.add_error("max_file_size", "File size is higher than the disk space") - if cleaned_data.get('new_worker_minimum_messages'): - if cleaned_data['new_worker_minimum_messages'] > cleaned_data.get('queue_size', 10000): - self.add_error("new_worker_minimum_messages", "This value cannot be over the queue size") return cleaned_data -class LogOMHIREDISForm(ModelForm): +class LogOMHIREDISForm(LogOMForm): - class Meta: + class Meta(LogOMForm.Meta): model = LogOMHIREDIS - fields = ('name', 'enabled', 'target', 'port', 'mode', 'key', 'dynamic_key', 'pwd', 'use_rpush', - 'expire_key', 'stream_outfield', 'stream_capacitylimit', 'send_as_raw', - 'queue_size', 'dequeue_size', 'queue_timeout_shutdown', 'max_workers', 'new_worker_minimum_messages', - 'worker_timeout_shutdown', 'enable_retry', 'enable_disk_assist', 'high_watermark', 'low_watermark', - 'max_file_size', 'max_disk_space') + fields = LogOMForm.Meta.fields + ('target', 'port', 'mode', 'key', 'dynamic_key', 'pwd', + 'use_rpush', 'expire_key', 'stream_outfield', 'stream_capacitylimit') widgets = { - 'enabled': CheckboxInput(attrs={'class': 'js-switch'}), - 'name': TextInput(attrs={'class': 'form-control'}), 'target': TextInput(attrs={'class': 'form-control'}), 'port': NumberInput(attrs={'class': 'form-control'}), - 'mode': Select(choices=OMHIREDIS_MODE_CHOICES, attrs={'class': 'form-control select2'}), + 'mode': Select(choices=OMHIREDIS_MODE_CHOICES, attrs={'class': 'select2'}), 'key': TextInput(attrs={'class': 'form-control'}), 'dynamic_key': CheckboxInput(attrs={'class': 'js-switch'}), 'pwd': TextInput(attrs={'class': 'form-control'}), @@ -220,34 +173,11 @@ class Meta: 'expire_key': NumberInput(attrs={'class': 'form-control'}), 'stream_outfield': TextInput(attrs={'class': 'form-control'}), 'stream_capacitylimit': NumberInput(attrs={'class': 'form-control'}), - 'send_as_raw': CheckboxInput(attrs={'class': 'form-control js-switch'}), - 'queue_size': NumberInput(attrs={'class': 'form-control'}), - 'dequeue_size': NumberInput(attrs={'class': 'form-control'}), - 'queue_timeout_shutdown': NumberInput(attrs={'class': 'form-control', 'placeholder': 10}), - 'max_workers': NumberInput(attrs={'class': 'form-control', 'placeholder': 1}), - 'new_worker_minimum_messages': NumberInput(attrs={'class': 'form-control', 'placeholder': 'queue size / max workers'}), - 'worker_timeout_shutdown': NumberInput(attrs={'class': 'form-control', 'placeholder': 60_000}), - 'enable_retry': CheckboxInput(attrs={'class': 'js-switch'}), - 'enable_disk_assist': CheckboxInput(attrs={'class': 'js-switch'}), - 'high_watermark': NumberInput(attrs={'class': 'form-control'}), - 'low_watermark': NumberInput(attrs={'class': 'form-control'}), - 'max_file_size': NumberInput(attrs={'class': 'form-control'}), - 'max_disk_space': NumberInput(attrs={'class': 'form-control'}) } - - def __init__(self, *args, **kwargs): - super().__init__(*args, **kwargs) - for field_name in ['high_watermark', 'low_watermark', 'max_file_size', 'max_disk_space']: - self.fields[field_name].required = False - - def clean_name(self): - field = self.cleaned_data.get('name') - if not field: - raise ValidationError("This field is required.") - return field.replace(' ', '_') + widgets.update(LogOMForm.Meta.widgets) def clean_key(self): - key = self.cleaned_data.get('key') + key = self.cleaned_data['key'] if " " in key: raise ValidationError("Cannot contain spaces") return key @@ -255,160 +185,66 @@ def clean_key(self): def clean(self): """ Verify needed fields - depending on mode chosen """ cleaned_data = super().clean() - if cleaned_data.get('enable_disk_assist') == True: - try: - assert cleaned_data.get('high_watermark'), "high_watermark" - assert cleaned_data.get('low_watermark'), "low_watermark" - except AssertionError as e: - self.add_error(str(e), "This field is required.") - else: - if cleaned_data.get('queue_size') and (cleaned_data.get('queue_size') < cleaned_data.get('high_watermark') or cleaned_data.get('queue_size') < cleaned_data.get('low_watermark')): - self.add_error("queue_size", "Queue size is lower than a watermark") - if cleaned_data.get('high_watermark') < cleaned_data.get('low_watermark'): - self.add_error("high_watermark", "High watermark is lower than the low watermark value") - self.add_error("low_watermark", "Low watermark is higher than the high watermark value") - try: - assert cleaned_data.get('max_file_size'), "max_file_size" - assert cleaned_data.get('max_disk_space'), "max_disk_space" - except AssertionError as e: - self.add_error(str(e), "This field is required.") - else: - if cleaned_data.get('max_file_size') > cleaned_data.get('max_disk_space'): - self.add_error("max_file_size", "File size is higher than the disk space") if cleaned_data.get('dynamic_key') == True and cleaned_data.get('key'): - key = cleaned_data.get('key') + key = cleaned_data['key'] if key.count("%") % 2 != 0: self.add_error("key", "seems like your number of '%' is incorrect, please check your templated key") - if cleaned_data.get('new_worker_minimum_messages'): - if cleaned_data['new_worker_minimum_messages'] > cleaned_data.get('queue_size', 10000): - self.add_error("new_worker_minimum_messages", "This value cannot be over the queue size") return cleaned_data -class LogOMFWDForm(ModelForm): +class LogOMFWDForm(LogOMForm): - class Meta: + class Meta(LogOMForm.Meta): model = LogOMFWD - fields = ('name', 'enabled', 'target', 'port', 'protocol', 'zip_level', 'queue_size', 'dequeue_size', - 'queue_timeout_shutdown', 'max_workers', 'new_worker_minimum_messages', 'worker_timeout_shutdown', - 'enable_retry', 'enable_disk_assist', 'high_watermark', 'low_watermark', 'max_file_size', - 'max_disk_space', 'ratelimit_interval', 'ratelimit_burst', 'send_as_raw') + fields = LogOMForm.Meta.fields + ('target', 'port', 'protocol', 'zip_level', + 'ratelimit_interval', 'ratelimit_burst') widgets = { - 'enabled': CheckboxInput(attrs={"class": " js-switch"}), - 'name': TextInput(attrs={'class': 'form-control'}), 'target': TextInput(attrs={'class': 'form-control'}), 'port': NumberInput(attrs={'class': 'form-control'}), - 'protocol': Select(choices=OMFWD_PROTOCOL, attrs={'class': 'form-control select2'}), + 'protocol': Select(choices=OMFWD_PROTOCOL, attrs={'class': 'select2'}), 'zip_level': NumberInput(attrs={'class': 'form-control'}), - 'queue_size': NumberInput(attrs={'class': 'form-control'}), - 'dequeue_size': NumberInput(attrs={'class': 'form-control'}), - 'queue_timeout_shutdown': NumberInput(attrs={'class': 'form-control', 'placeholder': 10}), - 'max_workers': NumberInput(attrs={'class': 'form-control', 'placeholder': 1}), - 'new_worker_minimum_messages': NumberInput(attrs={'class': 'form-control', 'placeholder': 'queue size / max workers'}), - 'worker_timeout_shutdown': NumberInput(attrs={'class': 'form-control', 'placeholder': 60_000}), - 'enable_retry': CheckboxInput(attrs={"class": " js-switch"}), - 'enable_disk_assist': CheckboxInput(attrs={"class": " js-switch"}), - 'high_watermark': NumberInput(attrs={'class': 'form-control'}), - 'low_watermark': NumberInput(attrs={'class': 'form-control'}), - 'max_file_size': NumberInput(attrs={'class': 'form-control'}), - 'max_disk_space': NumberInput(attrs={'class': 'form-control'}), 'ratelimit_interval': NumberInput(attrs={'class': 'form-control'}), 'ratelimit_burst': NumberInput(attrs={'class': 'form-control'}), - 'send_as_raw': CheckboxInput(attrs={'class': 'form-control js-switch'}) } - - def __init__(self, *args, **kwargs): - super().__init__(*args, **kwargs) - for field_name in ['high_watermark', 'low_watermark', 'max_file_size', 'max_disk_space']: - self.fields[field_name].required = False - - def clean_name(self): - field = self.cleaned_data.get('name') - if not field: - raise ValidationError("This field is required.") - return field.replace(' ', '_') + widgets.update(LogOMForm.Meta.widgets) def clean(self): """ Verify needed fields - depending on mode chosen """ cleaned_data = super().clean() - if cleaned_data.get('enable_disk_assist') == True: - try: - assert cleaned_data.get('high_watermark'), "high_watermark" - assert cleaned_data.get('low_watermark'), "low_watermark" - except AssertionError as e: - self.add_error(str(e), "This field is required.") - else: - if cleaned_data.get('queue_size') and (cleaned_data.get('queue_size') < cleaned_data.get('high_watermark') or cleaned_data.get('queue_size') < cleaned_data.get('low_watermark')): - self.add_error("queue_size", "Queue size is lower than a watermark") - if cleaned_data.get('high_watermark') < cleaned_data.get('low_watermark'): - self.add_error("high_watermark", "High watermark is lower than the low watermark value") - self.add_error("low_watermark", "Low watermark is higher than the high watermark value") - try: - assert cleaned_data.get('max_file_size'), "max_file_size" - assert cleaned_data.get('max_disk_space'), "max_disk_space" - except AssertionError as e: - self.add_error(str(e), "This field is required.") - else: - if cleaned_data.get('max_file_size') > cleaned_data.get('max_disk_space'): - self.add_error("max_file_size", "File size is higher than the disk space") """ if ratelimit_interval or ratelimit_burst is specified, the other cannot be left blank""" if cleaned_data.get('ratelimit_interval') and not cleaned_data.get('ratelimit_burst'): self.add_error("ratelimit_burst", "This field cannot be left blank if rate-limiting interval is set") if cleaned_data.get('ratelimit_burst') and not cleaned_data.get('ratelimit_interval'): self.add_error("ratelimit_interval", "This field cannot be left blank if rate-limiting burst is set") - if cleaned_data.get('new_worker_minimum_messages'): - if cleaned_data['new_worker_minimum_messages'] > cleaned_data.get('queue_size', 10000): - self.add_error("new_worker_minimum_messages", "This value cannot be over the queue size") return cleaned_data -class LogOMElasticSearchForm(ModelForm): +class LogOMElasticSearchForm(LogOMForm): - class Meta: + x509_certificate = ModelChoiceField( + queryset=X509Certificate.objects.filter(is_ca=False).only(*(X509Certificate.str_attrs())), + required=False, + widget=Select(attrs={'class': 'select2'}), + empty_label="No SSL" + ) + + class Meta(LogOMForm.Meta): model = LogOMElasticSearch - fields = ('name', 'enabled', 'servers', 'es8_compatibility', 'data_stream_mode', 'retry_on_els_failures', 'index_pattern', 'uid', 'pwd', - 'x509_certificate', 'send_as_raw', 'queue_size', 'dequeue_size', 'queue_timeout_shutdown', - 'max_workers', 'new_worker_minimum_messages', 'worker_timeout_shutdown', 'enable_retry', - 'enable_disk_assist', 'high_watermark', 'low_watermark', 'max_file_size', 'max_disk_space') + fields = LogOMForm.Meta.fields + ('servers', 'es8_compatibility', 'data_stream_mode', + 'retry_on_els_failures', 'index_pattern', 'uid', 'pwd', 'x509_certificate') widgets = { - 'enabled': CheckboxInput(attrs={"class": "js-switch"}), - 'name': TextInput(attrs={'class': 'form-control'}), 'servers': TextInput(attrs={'class': 'form-control'}), - 'es8_compatibility': CheckboxInput(attrs={"class": "js-switch"}), - 'data_stream_mode': CheckboxInput(attrs={"class": "js-switch"}), - 'retry_on_els_failures': CheckboxInput(attrs={"class": "js-switch"}), + 'es8_compatibility': CheckboxInput(attrs={'class': 'js-switch'}), + 'data_stream_mode': CheckboxInput(attrs={'class': 'js-switch'}), + 'retry_on_els_failures': CheckboxInput(attrs={'class': 'js-switch'}), 'index_pattern': TextInput(attrs={'class': 'form-control'}), 'uid': TextInput(attrs={'class': 'form-control'}), 'pwd': TextInput(attrs={'class': 'form-control'}), 'x509_certificate': Select(attrs={'class': 'form-control'}), - 'send_as_raw': CheckboxInput(attrs={'class': 'form-control js-switch'}), - 'queue_size': NumberInput(attrs={'class': 'form-control'}), - 'dequeue_size': NumberInput(attrs={'class': 'form-control'}), - 'queue_timeout_shutdown': NumberInput(attrs={'class': 'form-control', 'placeholder': 10}), - 'max_workers': NumberInput(attrs={'class': 'form-control', 'placeholder': 1}), - 'new_worker_minimum_messages': NumberInput(attrs={'class': 'form-control', 'placeholder': 'queue size / max workers'}), - 'worker_timeout_shutdown': NumberInput(attrs={'class': 'form-control', 'placeholder': 60_000}), - 'enable_retry': CheckboxInput(attrs={"class": "js-switch"}), - 'enable_disk_assist': CheckboxInput(attrs={"class": "js-switch"}), - 'high_watermark': NumberInput(attrs={'class': 'form-control'}), - 'low_watermark': NumberInput(attrs={'class': 'form-control'}), - 'max_file_size': NumberInput(attrs={'class': 'form-control'}), - 'max_disk_space': NumberInput(attrs={'class': 'form-control'}) } - - def __init__(self, *args, **kwargs): - super().__init__(*args, **kwargs) - self.fields['x509_certificate'].empty_label = "No TLS certificate" - for field_name in ['x509_certificate', 'high_watermark', 'low_watermark', 'max_file_size', 'max_disk_space']: - self.fields[field_name].required = False - - def clean_name(self): - field = self.cleaned_data.get('name') - if not field: - raise ValidationError("This field is required.") - return field.replace(' ', '_') + widgets.update(LogOMForm.Meta.widgets) def clean_index_pattern(self): field = self.cleaned_data.get('index_pattern') @@ -420,166 +256,62 @@ def clean(self): cleaned_data = super().clean() if cleaned_data.get('retry_on_els_failures') == True and cleaned_data.get('data_stream_mode') == False: self.add_error('retry_on_els_failures', "This field cannot be set if Stream Mode is disabled.") - if cleaned_data.get('enable_disk_assist') == True: - try: - assert cleaned_data.get('high_watermark'), "high_watermark" - assert cleaned_data.get('low_watermark'), "low_watermark" - except AssertionError as e: - self.add_error(str(e), "This field is required.") - else: - if cleaned_data.get('queue_size') and (cleaned_data.get('queue_size') < cleaned_data.get('high_watermark') or cleaned_data.get('queue_size') < cleaned_data.get('low_watermark')): - self.add_error("queue_size", "Queue size is lower than a watermark") - if cleaned_data.get('high_watermark') < cleaned_data.get('low_watermark'): - self.add_error("high_watermark", "High watermark is lower than the low watermark value") - self.add_error("low_watermark", "Low watermark is higher than the high watermark value") - try: - assert cleaned_data.get('max_file_size'), "max_file_size" - assert cleaned_data.get('max_disk_space'), "max_disk_space" - except AssertionError as e: - self.add_error(str(e), "This field is required.") - else: - if cleaned_data.get('max_file_size') > cleaned_data.get('max_disk_space'): - self.add_error("max_file_size", "File size is higher than the disk space") - if cleaned_data.get('new_worker_minimum_messages'): - if cleaned_data['new_worker_minimum_messages'] > cleaned_data.get('queue_size', 10000): - self.add_error("new_worker_minimum_messages", "This value cannot be over the queue size") return cleaned_data -class LogOMMongoDBForm(ModelForm): +class LogOMMongoDBForm(LogOMForm): x509_certificate = ModelChoiceField( queryset=X509Certificate.objects.filter(is_ca=False).only(*(X509Certificate.str_attrs())), required=False, - widget=Select(attrs={'class': 'form-control select2'}), + widget=Select(attrs={'class': 'select2'}), empty_label="No SSL" ) - class Meta: + class Meta(LogOMForm.Meta): model = LogOMMongoDB - fields = ('name', 'enabled', 'db', 'collection', 'uristr', 'x509_certificate', 'send_as_raw', 'queue_size', - 'dequeue_size', 'queue_timeout_shutdown', 'max_workers', 'new_worker_minimum_messages', - 'worker_timeout_shutdown', 'enable_retry', 'enable_disk_assist', 'high_watermark', 'low_watermark', - 'max_file_size', 'max_disk_space') + fields = LogOMForm.Meta.fields + ('db', 'collection', 'uristr', 'x509_certificate') widgets = { - 'enabled': CheckboxInput(attrs={"class": " js-switch"}), - 'name': TextInput(attrs={'class': 'form-control'}), 'db': TextInput(attrs={'class': 'form-control'}), 'collection': TextInput(attrs={'class': 'form-control'}), 'uristr': TextInput(attrs={'class': 'form-control'}), - 'send_as_raw': CheckboxInput(attrs={'class': 'form-control js-switch'}), - 'queue_size': NumberInput(attrs={'class': 'form-control'}), - 'dequeue_size': NumberInput(attrs={'class': 'form-control'}), - 'queue_timeout_shutdown': NumberInput(attrs={'class': 'form-control', 'placeholder': 10}), - 'max_workers': NumberInput(attrs={'class': 'form-control', 'placeholder': 1}), - 'new_worker_minimum_messages': NumberInput(attrs={'class': 'form-control', 'placeholder': 'queue size / max workers'}), - 'worker_timeout_shutdown': NumberInput(attrs={'class': 'form-control', 'placeholder': 60_000}), - 'enable_retry': CheckboxInput(attrs={"class": " js-switch"}), - 'enable_disk_assist': CheckboxInput(attrs={"class": " js-switch"}), - 'high_watermark': NumberInput(attrs={'class': 'form-control'}), - 'low_watermark': NumberInput(attrs={'class': 'form-control'}), - 'max_file_size': NumberInput(attrs={'class': 'form-control'}), - 'max_disk_space': NumberInput(attrs={'class': 'form-control'}) } - - def __init__(self, *args, **kwargs): - super().__init__(*args, **kwargs) - for field_name in ['high_watermark', 'low_watermark', 'max_file_size', 'max_disk_space']: - self.fields[field_name].required = False - - def clean_name(self): - field = self.cleaned_data.get('name') - if not field: - raise ValidationError("This field is required.") - return field.replace(' ', '_') - - def clean(self): - """ Verify needed fields - depending on mode chosen """ - cleaned_data = super().clean() - if cleaned_data.get('enable_disk_assist') == True: - try: - assert cleaned_data.get('high_watermark'), "high_watermark" - assert cleaned_data.get('low_watermark'), "low_watermark" - except AssertionError as e: - self.add_error(str(e), "This field is required.") - else: - if cleaned_data.get('queue_size') and (cleaned_data.get('queue_size') < cleaned_data.get('high_watermark') or cleaned_data.get('queue_size') < cleaned_data.get('low_watermark')): - self.add_error("queue_size", "Queue size is lower than a watermark") - if cleaned_data.get('high_watermark') < cleaned_data.get('low_watermark'): - self.add_error("high_watermark", "High watermark is lower than the low watermark value") - self.add_error("low_watermark", "Low watermark is higher than the high watermark value") - try: - assert cleaned_data.get('max_file_size'), "max_file_size" - assert cleaned_data.get('max_disk_space'), "max_disk_space" - except AssertionError as e: - self.add_error(str(e), "This field is required.") - else: - if cleaned_data.get('max_file_size') > cleaned_data.get('max_disk_space'): - self.add_error("max_file_size", "File size is higher than the disk space") - if cleaned_data.get('new_worker_minimum_messages'): - if cleaned_data['new_worker_minimum_messages'] > cleaned_data.get('queue_size', 10000): - self.add_error("new_worker_minimum_messages", "This value cannot be over the queue size") - return cleaned_data + widgets.update(LogOMForm.Meta.widgets) -class LogOMKafkaForm(ModelForm): +class LogOMKafkaForm(LogOMForm): - class Meta: + class Meta(LogOMForm.Meta): model = LogOMKAFKA - fields = ('name', 'enabled', 'broker', 'topic', 'key', 'dynaKey', 'dynaTopic', 'partitions_useFixed', - 'partitions_auto', 'confParam', 'topicConfParam', 'queue_size', 'dequeue_size', - 'queue_timeout_shutdown', 'max_workers', 'new_worker_minimum_messages', 'worker_timeout_shutdown', - 'enable_retry', 'enable_disk_assist', 'high_watermark', 'low_watermark', 'max_file_size', - 'max_disk_space') + fields = LogOMForm.Meta.fields + ('broker', 'topic', 'key', 'dynaKey', 'dynaTopic', 'partitions_useFixed', + 'partitions_auto', 'confParam', 'topicConfParam') widgets = { - 'enabled': CheckboxInput(attrs={"class": " js-switch"}), - 'name': TextInput(attrs={'class': 'form-control'}), 'broker': TextInput(attrs={'class': 'form-control'}), 'topic': TextInput(attrs={'class': 'form-control'}), 'key': TextInput(attrs={'class': 'form-control'}), - 'dynaKey': CheckboxInput(attrs={"class": " js-switch"}), - 'dynaTopic': CheckboxInput(attrs={"class": " js-switch"}), + 'dynaKey': CheckboxInput(attrs={'class': 'js-switch'}), + 'dynaTopic': CheckboxInput(attrs={'class': 'js-switch'}), 'partitions_useFixed': NumberInput(attrs={'class': 'form-control'}), - 'partitions_auto': CheckboxInput(attrs={"class": " js-switch"}), + 'partitions_auto': CheckboxInput(attrs={'class': 'js-switch'}), 'confParam': TextInput(attrs={'class': 'form-control', 'data-role': "tagsinput"}), 'topicConfParam': TextInput(attrs={'class': 'form-control', 'data-role': "tagsinput"}), - 'queue_size': NumberInput(attrs={'class': 'form-control'}), - 'dequeue_size': NumberInput(attrs={'class': 'form-control'}), - 'queue_timeout_shutdown': NumberInput(attrs={'class': 'form-control', 'placeholder': 10}), - 'max_workers': NumberInput(attrs={'class': 'form-control', 'placeholder': 1}), - 'new_worker_minimum_messages': NumberInput(attrs={'class': 'form-control', 'placeholder': 'queue size / max workers'}), - 'worker_timeout_shutdown': NumberInput(attrs={'class': 'form-control', 'placeholder': 60_000}), - 'enable_retry': CheckboxInput(attrs={"class": " js-switch"}), - 'enable_disk_assist': CheckboxInput(attrs={"class": " js-switch"}), - 'high_watermark': NumberInput(attrs={'class': 'form-control'}), - 'low_watermark': NumberInput(attrs={'class': 'form-control'}), - 'max_file_size': NumberInput(attrs={'class': 'form-control'}), - 'max_disk_space': NumberInput(attrs={'class': 'form-control'}) } + widgets.update(LogOMForm.Meta.widgets) def __init__(self, *args, **kwargs): super().__init__(*args, **kwargs) - for field_name in ['high_watermark', 'low_watermark', 'max_file_size', 'max_disk_space']: - self.fields[field_name].required = False - self.initial['confParam'] = ','.join(self.initial.get('confParam', []) or self.fields['confParam'].initial) self.initial['topicConfParam'] = ','.join(self.initial.get('topicConfParam', []) or self.fields['topicConfParam'].initial) - def clean_name(self): - field = self.cleaned_data.get('name') - if not field: - raise ValidationError("This field is required.") - return field.replace(' ', '_') - def clean_key(self): - key = self.cleaned_data.get('key') - if " " in key: + key = self.cleaned_data['key'] + if key is not None and " " in key: raise ValidationError("Cannot contain spaces") return key def clean_topic(self): - topic = self.cleaned_data.get('topic') + topic = self.cleaned_data['topic'] if " " in topic: raise ValidationError("Cannot contain spaces") return topic @@ -631,38 +363,15 @@ def clean_topicConfParam(self): def clean(self): """ Verify needed fields - depending on mode chosen """ cleaned_data = super().clean() - if cleaned_data.get('enable_disk_assist') == True: - try: - assert cleaned_data.get('high_watermark'), "high_watermark" - assert cleaned_data.get('low_watermark'), "low_watermark" - except AssertionError as e: - self.add_error(str(e), "This field is required.") - else: - if cleaned_data.get('queue_size') and (cleaned_data.get('queue_size') < cleaned_data.get('high_watermark') or cleaned_data.get('queue_size') < cleaned_data.get('low_watermark')): - self.add_error("queue_size", "Queue size is lower than a watermark") - if cleaned_data.get('high_watermark') < cleaned_data.get('low_watermark'): - self.add_error("high_watermark", "High watermark is lower than the low watermark value") - self.add_error("low_watermark", "Low watermark is higher than the high watermark value") - try: - assert cleaned_data.get('max_file_size'), "max_file_size" - assert cleaned_data.get('max_disk_space'), "max_disk_space" - except AssertionError as e: - self.add_error(str(e), "This field is required.") - else: - if cleaned_data.get('max_file_size') > cleaned_data.get('max_disk_space'): - self.add_error("max_file_size", "File size is higher than the disk space") if cleaned_data.get('dynaKey') == True: if cleaned_data.get('key'): - key = cleaned_data.get('key') + key = cleaned_data['key'] if key.count("%") % 2 != 0: self.add_error("key", "seems like your number of '%' is incorrect, please check your templated key") else: self.add_error("key", "This field is required.") if cleaned_data.get('dynaTopic') == True and cleaned_data.get('topic'): - topic = cleaned_data.get('topic') + topic = cleaned_data['topic'] if topic.count("%") % 2 != 0: self.add_error("topic", "seems like your number of '%' is incorrect, please check your templated topic") - if cleaned_data.get('new_worker_minimum_messages'): - if cleaned_data['new_worker_minimum_messages'] > cleaned_data.get('queue_size', 10000): - self.add_error("new_worker_minimum_messages", "This value cannot be over the queue size") - return cleaned_data \ No newline at end of file + return cleaned_data diff --git a/vulture_os/applications/logfwd/models.py b/vulture_os/applications/logfwd/models.py index 0f2eea0e..68cf81b0 100644 --- a/vulture_os/applications/logfwd/models.py +++ b/vulture_os/applications/logfwd/models.py @@ -89,6 +89,7 @@ class LogOM (models.Model): name = models.TextField(unique=True, blank=False, null=False, default="Log Output Module", help_text=_("Name of the Log Output Module")) internal = models.BooleanField(default=False, help_text=_("Is this LogForwarder internal")) + enabled = models.BooleanField(default=True) queue_size = models.PositiveIntegerField( default=10000, help_text=_("Size of the queue in nb of message"), @@ -141,32 +142,41 @@ class LogOM (models.Model): ) high_watermark = models.PositiveIntegerField( default=8000, - null=True, + null=False, help_text=_("Target of the high watermark"), verbose_name=_("High watermark target"), validators=[MinValueValidator(100)] ) low_watermark = models.PositiveIntegerField( default=6000, - null=True, + null=False, help_text=_("Set the value of the low watermark"), verbose_name=_("Low watermark target"), validators=[MinValueValidator(100)] ) max_file_size = models.IntegerField( default=256, - null=True, + null=False, help_text=_("Set the value of the queue in MB"), verbose_name=_("Max file size of the queue in MB"), validators=[MinValueValidator(1)] ) max_disk_space = models.IntegerField( - default=1024, - null=True, + default=0, + null=False, help_text=_("Limit the maximum disk space used by the queue in MB"), - verbose_name=_("Max disk space used by the queue in MB"), - validators=[MinValueValidator(1)] + verbose_name=_("Max disk space used by the queue in MB (set to zero to disable)"), + validators=[MinValueValidator(0)] ) + spool_directory = models.TextField( + default="/var/tmp", + null=False, + help_text=_("Defines an existing folder to store queue files into"), + verbose_name=_("Existing folder to store queue files to"), + validators=[RegexValidator( + regex=r"^/.*$", + message="Value should be a valid fullpath, beginning with a '/'" + )]) send_as_raw = models.BooleanField( default=False, help_text=_("Send logs without any modification"), @@ -281,6 +291,7 @@ def to_template(self, **kwargs): 'low_watermark': self.low_watermark, 'max_file_size': self.max_file_size, 'max_disk_space': self.max_disk_space, + 'spool_directory': self.spool_directory, } @@ -288,7 +299,6 @@ class LogOMFile(LogOM): file = models.TextField(null=False) flush_interval = models.IntegerField(default=1, null=False) async_writing = models.BooleanField(default=True) - enabled = models.BooleanField(default=True) retention_time = models.PositiveIntegerField(default=30, validators=[MinValueValidator(1)]) rotation_period = models.TextField(default=ROTATION_PERIOD_CHOICES[0][0], choices=ROTATION_PERIOD_CHOICES) @@ -313,6 +323,7 @@ def to_html_template(self): return { 'id': str(self.id), 'internal': self.internal, + 'enabled': self.enabled, 'name': self.name, 'type': 'File', 'output': self.file @@ -372,7 +383,6 @@ def __str__(self): class LogOMRELP(LogOM): target = models.TextField(null=False, default="1.2.3.4") port = models.IntegerField(null=False, default=514) - enabled = models.BooleanField(default=True) tls_enabled = models.BooleanField( default=True, help_text=_("If set to on, the RELP connection will be encrypted by TLS.") @@ -400,6 +410,7 @@ def to_html_template(self): return { 'id': str(self.id), 'internal': self.internal, + 'enabled': self.enabled, 'name': self.name, 'type': 'RELP', 'output': self.target + ':' + str(self.port) @@ -438,7 +449,6 @@ class LogOMHIREDIS(LogOM): key = models.TextField(null=False, default="MyKey") dynamic_key = models.BooleanField(default=False) pwd = models.TextField(blank=True, default=None) - enabled = models.BooleanField(default=True) use_rpush = models.BooleanField( default=False, blank=True, @@ -486,6 +496,7 @@ def to_html_template(self): return { 'id': str(self.id), 'internal': self.internal, + 'enabled': self.enabled, 'name': self.name, 'type': 'Redis', 'output': f"{self.target}:{self.port} ({'dynamic ' if self.dynamic_key else ''}key = {self.key})" @@ -532,7 +543,6 @@ class LogOMFWD(LogOM): help_text=_("Port on which to send logs to .") ) protocol = models.TextField(null=False, choices=OMFWD_PROTOCOL, default="tcp") - enabled = models.BooleanField(default=True) zip_level = models.PositiveIntegerField( default=0, validators=[MinValueValidator(0), MaxValueValidator(9)], @@ -553,6 +563,7 @@ def to_html_template(self): return { 'id': str(self.id), 'internal': self.internal, + 'enabled': self.enabled, 'name': self.name, 'type': 'Syslog', 'output': self.target + ':' + str(self.port) + ' ({})'.format(self.protocol) @@ -582,7 +593,6 @@ def get_rsyslog_template(self): class LogOMElasticSearch(LogOM): - enabled = models.BooleanField(default=True) servers = models.TextField(null=False, default='["https://els-1:9200", "https://els-2:9200"]') es8_compatibility = models.BooleanField( default=False, @@ -607,7 +617,8 @@ class LogOMElasticSearch(LogOM): on_delete=models.CASCADE, help_text=_("X509Certificate object to use."), default=None, - null=True + null=True, + blank=True ) def to_dict(self, fields=None): @@ -626,6 +637,7 @@ def to_html_template(self): 'id': str(self.id), 'name': self.name, 'internal': self.internal, + 'enabled': self.enabled, 'type': 'Elasticsearch', 'output': self.servers + ' (index = {})'.format(self.index_pattern) } @@ -696,7 +708,6 @@ class LogOMMongoDB(LogOM): db = models.TextField(unique=True, null=False, default='MyDatabase') collection = models.TextField(unique=True, null=False, default='MyLogs') uristr = models.TextField(null=False, default='mongodb://1.2.3.4:9091/?replicaset=Vulture&ssl=true') - enabled = models.BooleanField(default=True) x509_certificate = models.ForeignKey( X509Certificate, on_delete=models.CASCADE, @@ -716,6 +727,7 @@ def to_html_template(self): return { 'id': str(self.id), 'internal': self.internal, + 'enabled': self.enabled, 'name': self.name, 'type': 'MongoDB', 'output': self.uristr + ' (db = {})'.format(self.db) @@ -772,7 +784,6 @@ def mapping(self): class LogOMKAFKA(LogOM): broker = models.TextField(blank=True, default='["1.2.3.4:9092"]') - enabled = models.BooleanField(default=True) topic = models.TextField() key = models.TextField(blank=True) dynaKey = models.BooleanField(default=False) @@ -798,6 +809,7 @@ def to_html_template(self): return { 'id': str(self.id), 'internal': self.internal, + 'enabled': self.enabled, 'name': self.name, 'type': 'Kafka', 'output': self.broker + ' (topic = {})'.format(self.topic) diff --git a/vulture_os/applications/migrations/0024_factorize_logom_enabled.py b/vulture_os/applications/migrations/0024_factorize_logom_enabled.py new file mode 100644 index 00000000..b93106d8 --- /dev/null +++ b/vulture_os/applications/migrations/0024_factorize_logom_enabled.py @@ -0,0 +1,127 @@ +# Generated by Django 4.2.11 on 2024-03-15 05:27 + +import django.core.validators +from django.db import migrations, models +import djongo.models.fields + +enabled_logom = {} + +def save_enabled_state(apps, schema_editor): + logomfile_model = apps.get_model("applications", "LogOMFile") + logomrelp_model = apps.get_model("applications", "LogOMRELP") + logomhiredis_model = apps.get_model("applications", "LogOMHIREDIS") + logomfwd_model = apps.get_model("applications", "LogOMFWD") + logomelastic_model = apps.get_model("applications", "LogOMElasticSearch") + logommongodb_model = apps.get_model("applications", "LogOMMongoDB") + logomkafka_model = apps.get_model("applications", "LogOMKAFKA") + db_alias = schema_editor.connection.alias + logomfile = logomfile_model.objects.using(db_alias) + logomrelp = logomrelp_model.objects.using(db_alias) + logomhiredis = logomhiredis_model.objects.using(db_alias) + logomfwd = logomfwd_model.objects.using(db_alias) + logomelastic = logomelastic_model.objects.using(db_alias) + logommongodb = logommongodb_model.objects.using(db_alias) + logomkafka = logomkafka_model.objects.using(db_alias) + + for filefwd in logomfile.all(): + enabled_logom[filefwd.name] = filefwd.enabled + for logomrelp in logomrelp.all(): + enabled_logom[logomrelp.name] = logomrelp.enabled + for logomhiredis in logomhiredis.all(): + enabled_logom[logomhiredis.name] = logomhiredis.enabled + for logomfwd in logomfwd.all(): + enabled_logom[logomfwd.name] = logomfwd.enabled + for logomelastic in logomelastic.all(): + enabled_logom[logomelastic.name] = logomelastic.enabled + for logommongodb in logommongodb.all(): + enabled_logom[logommongodb.name] = logommongodb.enabled + for logomkafka in logomkafka.all(): + enabled_logom[logomkafka.name] = logomkafka.enabled + +def restore_enabled_state(apps, schema_editor): + logfwd_model = apps.get_model("applications", "LogOM") + db_alias = schema_editor.connection.alias + logfwd = logfwd_model.objects.using(db_alias) + + for fwd in logfwd.all(): + fwd.enabled = enabled_logom[fwd.name] + fwd.save() + +class Migration(migrations.Migration): + + dependencies = [ + ('applications', '0023_logomhiredis_expire_key_logomhiredis_mode_and_more'), + ] + + operations = [ + migrations.RunPython(save_enabled_state, migrations.RunPython.noop), + migrations.RemoveField( + model_name='logomelasticsearch', + name='enabled', + ), + migrations.RemoveField( + model_name='logomfile', + name='enabled', + ), + migrations.RemoveField( + model_name='logomfwd', + name='enabled', + ), + migrations.RemoveField( + model_name='logomhiredis', + name='enabled', + ), + migrations.RemoveField( + model_name='logomkafka', + name='enabled', + ), + migrations.RemoveField( + model_name='logommongodb', + name='enabled', + ), + migrations.RemoveField( + model_name='logomrelp', + name='enabled', + ), + migrations.AddField( + model_name='logom', + name='enabled', + field=models.BooleanField(default=True), + ), + migrations.RunPython(restore_enabled_state, migrations.RunPython.noop), + migrations.AddField( + model_name='logom', + name='spool_directory', + field=models.TextField(null=False, default='/var/tmp', help_text='Defines an existing folder to store queue files into', validators=[django.core.validators.RegexValidator(message="Value should be a valid fullpath, beginning with a '/'", regex='^/.*$')], verbose_name='Existing folder to store queue files to'), + ), + migrations.AddField( + model_name='logomelasticsearch', + name='retry_on_els_failures', + field=models.BooleanField(default=False, help_text="Let Rsyslog's Elasticsearch module handle and retry insertion failure", verbose_name='Handle failures and retries on ELS insertion'), + ), + migrations.AlterField( + model_name='logomelasticsearch', + name='x509_certificate', + field=models.ForeignKey(default=None, help_text='X509Certificate object to use.', null=True, on_delete=django.db.models.deletion.CASCADE, to='system.x509certificate'), + ), + migrations.AlterField( + model_name='logom', + name='high_watermark', + field=models.PositiveIntegerField(null=False, default=8000, help_text='Target of the high watermark', validators=[django.core.validators.MinValueValidator(100)], verbose_name='High watermark target'), + ), + migrations.AlterField( + model_name='logom', + name='low_watermark', + field=models.PositiveIntegerField(null=False, default=6000, help_text='Set the value of the low watermark', validators=[django.core.validators.MinValueValidator(100)], verbose_name='Low watermark target'), + ), + migrations.AlterField( + model_name='logom', + name='max_disk_space', + field=models.IntegerField(null=False, default=0, help_text='Limit the maximum disk space used by the queue in MB', validators=[django.core.validators.MinValueValidator(0)], verbose_name='Max disk space used by the queue in MB (set to zero to disable)'), + ), + migrations.AlterField( + model_name='logom', + name='max_file_size', + field=models.IntegerField(null=False, default=256, help_text='Set the value of the queue in MB', validators=[django.core.validators.MinValueValidator(1)], verbose_name='Max file size of the queue in MB'), + ), + ] diff --git a/vulture_os/applications/templates/apps/logfwd.html b/vulture_os/applications/templates/apps/logfwd.html index 939ec308..aab5afba 100644 --- a/vulture_os/applications/templates/apps/logfwd.html +++ b/vulture_os/applications/templates/apps/logfwd.html @@ -53,26 +53,45 @@

 {% translate "Logs Fo }, { sTitle: '{% translate "Name" %}', - name: "name", aTargets: [1], + name: "name", + aTargets: [1], defaultContent: "", - mData: "name" + mData: "name", + mRender: function(data, type, row) { + var result = ""; + switch( row.enabled ) { + case true: + result += ' '; + break; + case false: + result += ' '; + break; + default: + result += ' '; + break; + } + return result + data; + } }, { sTitle: '{% translate "Internal" %}', name: "internal", aTargets: [2], defaultContent: "", - mData: "internal" + mData: "internal", + bVisible: false }, { sTitle: '{% translate "Type" %}', - name: "type", aTargets: [3], + name: "type", + aTargets: [3], defaultContent: "", mData: "type", }, { sTitle: '{% translate "Output" %}', - name: "output", aTargets: [4], + name: "output", + aTargets: [4], defaultContent: "", mData: "output" }, diff --git a/vulture_os/applications/templates/apps/logfwd_LogOMElasticSearch.html b/vulture_os/applications/templates/apps/logfwd_LogOMElasticSearch.html index e72e793a..a169abd5 100644 --- a/vulture_os/applications/templates/apps/logfwd_LogOMElasticSearch.html +++ b/vulture_os/applications/templates/apps/logfwd_LogOMElasticSearch.html @@ -319,6 +319,17 @@

Form errors

+
+
+
+ +
+ {{form.spool_directory}} + {{form.spool_directory.errors|safe}} +
+
+
+
diff --git a/vulture_os/applications/templates/apps/logfwd_LogOMFWD.html b/vulture_os/applications/templates/apps/logfwd_LogOMFWD.html index 0c92d0d1..c29485ca 100644 --- a/vulture_os/applications/templates/apps/logfwd_LogOMFWD.html +++ b/vulture_os/applications/templates/apps/logfwd_LogOMFWD.html @@ -296,6 +296,17 @@

Form errors

+
+
+
+ +
+ {{form.spool_directory}} + {{form.spool_directory.errors|safe}} +
+
+
+
diff --git a/vulture_os/applications/templates/apps/logfwd_LogOMFile.html b/vulture_os/applications/templates/apps/logfwd_LogOMFile.html index 5925a25e..6fed46bc 100644 --- a/vulture_os/applications/templates/apps/logfwd_LogOMFile.html +++ b/vulture_os/applications/templates/apps/logfwd_LogOMFile.html @@ -295,6 +295,17 @@

 {% translate "Local F +
+
+
+ +
+ {{form.spool_directory}} + {{form.spool_directory.errors|safe}} +
+
+
+
diff --git a/vulture_os/applications/templates/apps/logfwd_LogOMHIREDIS.html b/vulture_os/applications/templates/apps/logfwd_LogOMHIREDIS.html index 907e2a5d..ad4c4076 100644 --- a/vulture_os/applications/templates/apps/logfwd_LogOMHIREDIS.html +++ b/vulture_os/applications/templates/apps/logfwd_LogOMHIREDIS.html @@ -341,6 +341,17 @@

Form errors

+
+
+
+ +
+ {{form.spool_directory}} + {{form.spool_directory.errors|safe}} +
+
+
+
diff --git a/vulture_os/applications/templates/apps/logfwd_LogOMKAFKA.html b/vulture_os/applications/templates/apps/logfwd_LogOMKAFKA.html index c4dc52c7..d6b9fd19 100644 --- a/vulture_os/applications/templates/apps/logfwd_LogOMKAFKA.html +++ b/vulture_os/applications/templates/apps/logfwd_LogOMKAFKA.html @@ -319,6 +319,17 @@

Form errors

+
+
+
+ +
+ {{form.spool_directory}} + {{form.spool_directory.errors|safe}} +
+
+
+
diff --git a/vulture_os/applications/templates/apps/logfwd_LogOMMongoDB.html b/vulture_os/applications/templates/apps/logfwd_LogOMMongoDB.html index 9d91697b..4d386073 100644 --- a/vulture_os/applications/templates/apps/logfwd_LogOMMongoDB.html +++ b/vulture_os/applications/templates/apps/logfwd_LogOMMongoDB.html @@ -284,6 +284,17 @@

 {% translate "MongoDB +
+
+
+ +
+ {{form.spool_directory}} + {{form.spool_directory.errors|safe}} +
+
+
+
diff --git a/vulture_os/applications/templates/apps/logfwd_LogOMRELP.html b/vulture_os/applications/templates/apps/logfwd_LogOMRELP.html index d7165a76..7a50a143 100644 --- a/vulture_os/applications/templates/apps/logfwd_LogOMRELP.html +++ b/vulture_os/applications/templates/apps/logfwd_LogOMRELP.html @@ -275,6 +275,17 @@

Form errors

+
+
+
+ +
+ {{form.spool_directory}} + {{form.spool_directory.errors|safe}} +
+
+
+