[rubrik] Enable TSDB (#12917)

gpop63 · ritalwar · web-flow · commit 236d1d1e1850 · 2025-03-04T12:16:02.000+02:00
* dont set timestamp in node_statistics * enable tsdb for data streams * bump package version * fix pipeline * Update packages/rubrik/changelog.yml Co-authored-by: Richa Talwar <102972658+ritalwar@users.noreply.github.com> * Update packages/rubrik/changelog.yml Co-authored-by: Richa Talwar <102972658+ritalwar@users.noreply.github.com> * fix changelog * Revert "dont set timestamp in node_statistics" This reverts commit 95a8c2f. * Revert "fix pipeline" This reverts commit c1bcaaa. * change interval to 1h * add time range filter in query * improve docs --------- Co-authored-by: Richa Talwar <102972658+ritalwar@users.noreply.github.com>
diff --git a/packages/rubrik/_dev/build/docs/README.md b/packages/rubrik/_dev/build/docs/README.md
@@ -145,6 +145,8 @@ Please refer to the following [document](https://www.elastic.co/guide/en/ecs/cur
 
 The `node_statistics` dataset provides metrics related to the performance of the Rubrik cluster nodes.
 
+**IMPORTANT: Setting `interval` to more than `1h` may cause documents to be dropped if node statistics metrics fall outside the index time range.**
+
 **ECS Field Reference**
 
 Please refer to the following [document](https://www.elastic.co/guide/en/ecs/current/ecs-field-reference.html) for detailed information on ECS fields.
diff --git a/packages/rubrik/changelog.yml b/packages/rubrik/changelog.yml
@@ -1,4 +1,12 @@
 # newer versions go on top
+- version: "0.8.0"
+  changes:
+    - description: |
+        Enable TSDB.
+        Added support for specifying a time range in `ClusterNodeStatistics` query.
+        Adjusted default interval from `24h` to `1h`.
+      type: enhancement
+      link: https://github.com/elastic/integrations/pull/12917
 - version: "0.7.0"
   changes:
     - description: Add `sla_domains` data stream
diff --git a/packages/rubrik/data_stream/drives/manifest.yml b/packages/rubrik/data_stream/drives/manifest.yml
@@ -70,3 +70,5 @@ streams:
         required: false
         show_user: false
         description: The request tracer logs requests and responses to the agent's local file-system for debugging configurations. Enabling this request tracing compromises security and should only be used for debugging. See [documentation](https://www.elastic.co/guide/en/beats/filebeat/current/filebeat-input-httpjson.html#_request_tracer_filename) for details.
+elasticsearch:
+  index_mode: time_series
diff --git a/packages/rubrik/data_stream/filesets/manifest.yml b/packages/rubrik/data_stream/filesets/manifest.yml
@@ -63,3 +63,5 @@ streams:
         default: "# filter:\n#   - field: \"location\"\n#     texts: \n#       - \"prod-lab.local\"\n"
         description: >-
           Specify filters for refining the data. Filters must be defined as an array of Filter objects. Refer to the [Rubrik API documentation](https://rubrikinc.github.io/rubrik-api-documentation/schema/reference/filter.doc.html) for valid fields.
+elasticsearch:
+  index_mode: time_series
diff --git a/packages/rubrik/data_stream/global_cluster_performance/manifest.yml b/packages/rubrik/data_stream/global_cluster_performance/manifest.yml
@@ -54,3 +54,5 @@ streams:
         required: false
         show_user: false
         description: The number of results per page in Rubrik GraphQL queries. Smaller pages mean more API requests but can be more efficient in some cases.
+elasticsearch:
+  index_mode: time_series
diff --git a/packages/rubrik/data_stream/managed_volumes/manifest.yml b/packages/rubrik/data_stream/managed_volumes/manifest.yml
@@ -70,3 +70,5 @@ streams:
         required: false
         show_user: false
         description: The request tracer logs requests and responses to the agent's local file-system for debugging configurations. Enabling this request tracing compromises security and should only be used for debugging. See [documentation](https://www.elastic.co/guide/en/beats/filebeat/current/filebeat-input-httpjson.html#_request_tracer_filename) for details.
+elasticsearch:
+  index_mode: time_series
diff --git a/packages/rubrik/data_stream/monitoring_jobs/manifest.yml b/packages/rubrik/data_stream/monitoring_jobs/manifest.yml
@@ -54,3 +54,5 @@ streams:
         required: false
         show_user: false
         description: The request tracer logs requests and responses to the agent's local file-system for debugging configurations. Enabling this request tracing compromises security and should only be used for debugging. See [documentation](https://www.elastic.co/guide/en/beats/filebeat/current/filebeat-input-httpjson.html#_request_tracer_filename) for details.
+elasticsearch:
+  index_mode: time_series
diff --git a/packages/rubrik/data_stream/mssql_databases/manifest.yml b/packages/rubrik/data_stream/mssql_databases/manifest.yml
@@ -78,3 +78,5 @@ streams:
         required: false
         show_user: false
         description: The request tracer logs requests and responses to the agent's local file-system for debugging configurations. Enabling this request tracing compromises security and should only be used for debugging. See [documentation](https://www.elastic.co/guide/en/beats/filebeat/current/filebeat-input-httpjson.html#_request_tracer_filename) for details.
+elasticsearch:
+  index_mode: time_series
diff --git a/packages/rubrik/data_stream/node_statistics/agent/stream/cel.yml.hbs b/packages/rubrik/data_stream/node_statistics/agent/stream/cel.yml.hbs
@@ -35,17 +35,18 @@ processors:
 
 state:
     index: 0
+    period: {{interval}}
     pageSize: {{pageSize}}
     queries:
       - |
-        query ClusterNodeStatistics($first: Int, $after: String) {
+        query ClusterNodeStatistics($first: Int, $after: String, $range: TimeRangeInput) {
           allClusterConnection(first: $first, after: $after) {
             count
             nodes {
               name
               type
               id
-              clusterNodeStats {
+              clusterNodeStats(timeRange: $range) {
                 nodeId
                 clusterPhysicalDataIngest
                 networkBytesReceived
@@ -81,6 +82,10 @@ program: |-
           "variables": {
             "first": has(state.pageSize) ? state.pageSize : null,
             "after": has(state.cursor) ? state.cursor.after : null,
+            "range": {
+              "start": now - duration(state.period),
+              "end": now.format(time_layout.RFC3339),
+            }
           }
         }.encode_json()
       }
@@ -132,7 +137,7 @@ program: |-
           "index": body.data.allClusterConnection.pageInfo.hasNextPage ? int(state.index) : (int(state.index) + 1) % size(state.queries),
           "cursor": {
             "after": body.data.allClusterConnection.pageInfo.hasNextPage ? body.data.allClusterConnection.pageInfo.endCursor : null
-          }
+          },
         }
       )
       :
diff --git a/packages/rubrik/data_stream/node_statistics/manifest.yml b/packages/rubrik/data_stream/node_statistics/manifest.yml
@@ -21,7 +21,7 @@ streams:
         description: The interval at which the API is polled, supported in seconds, minutes, and hours.
         show_user: true
         required: true
-        default: 24h
+        default: 1h
       - name: processors
         type: yaml
         title: Processors
@@ -54,3 +54,5 @@ streams:
         required: false
         show_user: false
         description: The number of results per page in Rubrik GraphQL queries. Smaller pages mean more API requests but can be more efficient in some cases.
+elasticsearch:
+  index_mode: time_series
diff --git a/packages/rubrik/data_stream/physical_hosts/manifest.yml b/packages/rubrik/data_stream/physical_hosts/manifest.yml
@@ -78,3 +78,5 @@ streams:
         required: false
         show_user: false
         description: The request tracer logs requests and responses to the agent's local file-system for debugging configurations. Enabling this request tracing compromises security and should only be used for debugging. See [documentation](https://www.elastic.co/guide/en/beats/filebeat/current/filebeat-input-httpjson.html#_request_tracer_filename) for details.
+elasticsearch:
+  index_mode: time_series
diff --git a/packages/rubrik/data_stream/tasks/manifest.yml b/packages/rubrik/data_stream/tasks/manifest.yml
@@ -47,3 +47,5 @@ streams:
         type: bool
         multi: false
         default: false
+elasticsearch:
+  index_mode: time_series
diff --git a/packages/rubrik/data_stream/virtual_machines/manifest.yml b/packages/rubrik/data_stream/virtual_machines/manifest.yml
@@ -70,3 +70,5 @@ streams:
         required: false
         show_user: false
         description: The request tracer logs requests and responses to the agent's local file-system for debugging configurations. Enabling this request tracing compromises security and should only be used for debugging. See [documentation](https://www.elastic.co/guide/en/beats/filebeat/current/filebeat-input-httpjson.html#_request_tracer_filename) for details.
+elasticsearch:
+  index_mode: time_series
diff --git a/packages/rubrik/docs/README.md b/packages/rubrik/docs/README.md
@@ -1058,6 +1058,8 @@ An example event for `global_cluster_performance` looks as following:
 
 The `node_statistics` dataset provides metrics related to the performance of the Rubrik cluster nodes.
 
+**IMPORTANT: Setting `interval` to more than `1h` may cause documents to be dropped if node statistics metrics fall outside the index time range.**
+
 **ECS Field Reference**
 
 Please refer to the following [document](https://www.elastic.co/guide/en/ecs/current/ecs-field-reference.html) for detailed information on ECS fields.
diff --git a/packages/rubrik/manifest.yml b/packages/rubrik/manifest.yml
@@ -1,7 +1,7 @@
 format_version: 3.0.2
 name: rubrik
 title: "Rubrik RSC Metrics"
-version: 0.7.0
+version: 0.8.0
 source:
   license: "Elastic-2.0"
 description: "Collect Metrics from Rubrik RSC with Elastic Agent."
