Fixing filebeat holding onto old file descriptors

Under certain scenariosFilebeat may hold onto old
log file descriptors. There are various settings
on Filebeat to prevent this from happening. This
patch adds to the charm three key settings that
help prevent this issue:

- ignore_older
- clean_inactive

Fixing: [canonical#87](canonical#87)

config.yaml

@@ -49,3 +49,20 @@ options:
     default: ""
     description: |
       A YAML list which will be injected to define additional prospectors/inputs.
+  ignore_older:
+    type: int
+    default: 0
+    description: |
+      If this option is enabled, Filebeat ignores any files that were modified before the specified timespan.
+      Configuring ignore_older can be especially useful if you keep log files for a long time. The passed 
+      values should be integers, e.g., "90", "3600", "600". The values are assumed to be in seconds
+      https://www.elastic.co/guide/en/beats/filebeat/current/filebeat-input-log.html#filebeat-input-log-ignore-older"
+  clean_inactive:
+    type: int
+    default: 0
+    description: |
+      When this option is enabled, Filebeat removes the state of a file after the specified period of inactivity has
+      elapsed. Bear in mind that if clean_inactive is enabled, it must be greater than ignore_older + scan_frequency to
+      make sure that no states are removed while a file is still being harvested. The passed values should be integers, 
+      e.g., "90", "3600", "600". The values are assumed to be in seconds:
+      https://www.elastic.co/guide/en/beats/filebeat/current/filebeat-input-log.html#filebeat-input-log-clean-inactive"

reactive/filebeat.py

@@ -76,6 +76,10 @@ def render_filebeat_template():
             )
         log(msg)
 
+    # run the sanity checks, abort the rendering if config is not valid
+    if not is_config_valid():
+        return
+
     # The v5 template is compatible with all versions < 6
     major = charms.apt.get_package_version("filebeat")[0]
     version = major if major.isdigit() and int(major) > 5 else "5"
@@ -193,6 +197,35 @@ def check_filebeat_repo():
     remove_state("filebeat.repo.changed")
 
 
+@when("config.changed")
+def is_config_valid() -> bool:
+    """Sanity check config options.
+
+    :return: True if all checks pass, False otherwise
+    """
+    log("Executing sanity checks on config values.")
+
+    # sanity check value compliance (e.g., clean_inactive must be greater
+    # than ignore_older + scan_frequency). scan_frequency is no longer a
+    # charm config option but it defaults to 10s and it's set to that in
+    # the charm templates so using that below.
+    scan_frequency = 10
+    ignore_older = config().get("ignore_older")
+    clean_inactive = config().get("clean_inactive")
+
+    if clean_inactive and clean_inactive <= ignore_older + scan_frequency:
+        log(
+            "Invalid config, make sure that the value for 'clean_inactive' "
+            "is greater than 'ignore_older + scan_frequency'."
+        )
+        status.blocked("Invalid config. Please check the logs for details.")
+        return False
+
+    log("Charm config sanity checks passed. Restarting Filebeat")
+    service("restart", "filebeat")
+    return True
+
+
 @hook("stop")
 def remove_filebeat():
     """Stop, purge, and remove all traces of filebeat."""

templates/filebeat-5.yml

@@ -16,6 +16,8 @@ filebeat:
       scan_frequency: 10s
       harvester_buffer_size: {{ harvester_buffer_size }}
       max_bytes: {{ max_bytes }}
+      ignore_older: {{ ignore_older }}
+      clean_inactive: {{ clean_inactive }}
       fields:
         juju_model_name: {{ juju_model_name }}
         juju_model_uuid: {{ juju_model_uuid }}
@@ -34,6 +36,8 @@ filebeat:
       scan_frequency: 10s
       harvester_buffer_size: {{ harvester_buffer_size }}
       max_bytes: {{ max_bytes }}
+      ignore_older: {{ ignore_older }}
+      clean_inactive: {{ clean_inactive }}
       fields_under_root: true
       symlinks: true
       # NB: disable json decoding; setting any of these would result in the

templates/filebeat-6.yml

@@ -17,6 +17,8 @@ filebeat:
       scan_frequency: 10s
       harvester_buffer_size: {{ harvester_buffer_size }}
       max_bytes: {{ max_bytes }}
+      ignore_older: {{ ignore_older }}
+      clean_inactive: {{ clean_inactive }}
       fields_under_root: false
       fields:
         type: logpath-logs
@@ -39,6 +41,8 @@ filebeat:
       scan_frequency: 10s
       harvester_buffer_size: {{ harvester_buffer_size }}
       max_bytes: {{ max_bytes }}
+      ignore_older: {{ ignore_older }}
+      clean_inactive: {{ clean_inactive }}
       fields_under_root: false
       fields:
         type: container-logs

templates/filebeat-7.yml

@@ -16,6 +16,8 @@ filebeat:
       scan_frequency: 10s
       harvester_buffer_size: {{ harvester_buffer_size }}
       max_bytes: {{ max_bytes }}
+      ignore_older: {{ ignore_older }}
+      clean_inactive: {{ clean_inactive }}
       fields_under_root: false
       fields:
         type: logpath-logs