[filestream] add support to decompress gzip files (#15047)

* filestream: add option to enabled decompressing GZIP files when ingesting them

Author: AndersonQ

packages/filestream/agent/input/filestream.yml.hbs

@@ -10,6 +10,10 @@ paths:
   - {{path}}
 {{/each}}
 
+{{#if gzip_experimental}}
+gzip_experimental: true
+{{/if}}
+
 {{#if pipeline}}
 pipeline: {{pipeline}}
 {{/if}}

packages/filestream/changelog.yml

@@ -1,4 +1,9 @@
 # newer versions go on top
+- version: "2.1.0"
+  changes:
+    - description: Allow filestream to be configured to ingest GZIP files
+      type: enhancement
+      link: https://github.com/elastic/integrations/pull/15047
 - version: "2.0.1"
   changes:
     - description: Remove hard-coded dataset field

packages/filestream/manifest.yml

@@ -3,7 +3,7 @@ name: filestream
 title: Custom Logs (Filestream)
 description: Collect log data using filestream with Elastic Agent.
 type: input
-version: 2.0.1
+version: 2.1.0
 conditions:
   kibana:
     version: "^9.2.0"
@@ -29,6 +29,15 @@ policy_templates:
         show_user: true
         default:
           - /var/log/*.log
+      - name: gzip_experimental
+        type: bool
+        title: Decompress GZIP files
+        multi: false
+        required: false
+        show_user: true
+        default: false
+        description: |
+          When enabled, filestream will decompress GZIP-compressed files (.gz) as they are read. This is beta feature. For full details, see the [documentation](https://www.elastic.co/guide/en/beats/filebeat/current/filebeat-input-filestream#reading-gzip-files). If enabled, you **must** also remove the '\.gz$' pattern from the "Exclude Files" setting to ensure `.gz` files are ingested. Available for Elastic Agent 9.2.0 and newer.
       - name: use_logs_stream
         type: bool
         title: Use the "logs" data stream
@@ -71,7 +80,7 @@ policy_templates:
         type: text
         title: Exclude Files
         description: |
-          A list of regular expressions to match the files that you want Elastic Agent to ignore. By default no files are excluded.
+          A list of regular expressions used to ignore files. By default, this list includes '\.gz$' to exclude GZIP files. If you enable the "Decompress GZIP files" option (Elastic Agent 9.2.0+), the default pattern, '\.gz$', **must** be removed.
         required: false
         show_user: true
         multi: true