Added Last Active Clients advanced statistics to Grafana Source Server

grr/server/grr_response_server/bin/grrafana.py

@@ -19,6 +19,9 @@
 from grr_response_core import config
 from grr_response_core.config import contexts
 from grr_response_core.config import server as config_server
+from grr_response_core.lib import rdfvalue
+from grr_response_core.lib.rdfvalues import stats as rdf_stats
+from grr_response_server import client_report_utils
 from grr_response_server import data_store
 from grr_response_server import fleet_utils
 from grr_response_server import fleetspeak_connector
@@ -29,6 +32,7 @@
 
 JSON_MIME_TYPE = "application/json"
 _FLEET_BREAKDOWN_DAY_BUCKETS = frozenset([1, 7, 14, 30])
+_LAST_ACTIVE_DAY_BUCKETS = frozenset([1, 3, 7, 30, 60])
 
 flags.DEFINE_bool(
     "version",
@@ -100,9 +104,9 @@ def __init__(
     self._record_values_extract_fn = record_values_extract_fn
 
   # Note: return type error issues at python/mypy#3915, python/typing#431
-  def ProcessQuery(
+  def ProcessQuery(  # type: ignore[override]
       self,
-      req: JSONRequest) -> _TargetWithDatapoints:  # type: ignore[override]
+      req: JSONRequest) -> _TargetWithDatapoints:
     client_id = req["scopedVars"]["ClientID"]["value"]
     start_range_ts = TimeToProtoTimestamp(req["range"]["from"])
     end_range_ts = TimeToProtoTimestamp(req["range"]["to"])
@@ -133,8 +137,8 @@ def __init__(self, name: str,
     self._days_active = days_active
 
   # Note: return type error issues at python/mypy#3915, python/typing#431
-  def ProcessQuery(
-      self, req: JSONRequest) -> _TableQueryResult:  # type: ignore[override]
+  def ProcessQuery(  # type: ignore[override]
+                   self, req: JSONRequest) -> _TableQueryResult:
     fleet_stats = self._get_fleet_stats_fn(_FLEET_BREAKDOWN_DAY_BUCKETS)
     totals = fleet_stats.GetTotalsForDay(self._days_active)
     return _TableQueryResult(
@@ -149,6 +153,39 @@ def ProcessQuery(
         type="table")
 
 
+class LastActiveMetric(Metric):
+  """A metric that represents the breakdown of client count based on the
+  last activity time of the client."""
+
+  def __init__(self,
+               name: str,
+               days_active: int) -> None:
+    super().__init__(name)
+    self._days_active = days_active
+
+  # TODO: Return type error issues at python/mypy#3915, python/typing#431
+  # TODO: This function is not clean and "translates" legacy graph data
+  # from the graphs datastore to Grafana-readable datapoints.
+  # We should aim to replace this once this datastore is deprecated.
+  # The current implementation is a modified version of LastActiveReportPlugin
+  # in grr/server/grr_response_server/gui/api_plugins/report_plugins/client_report_plugins.py
+  def ProcessQuery(self, req: JSONRequest) -> _TargetWithDatapoints: # type: ignore[override]
+    series_with_timestamps = client_report_utils.FetchAllGraphSeries(
+        label="All",
+        report_type=rdf_stats.ClientGraphSeries.ReportType.N_DAY_ACTIVE,
+        period=rdfvalue.Duration.From(180, rdfvalue.DAYS))
+    datapoints = []
+    for timestamp, graph_series in sorted(series_with_timestamps.items()):
+      for sample in graph_series.graphs[0]:
+        # Provide the time in js timestamps (milliseconds since the epoch).
+        days = sample.x_value // 1000000 // 24 // 60 // 60
+        if days == self._days_active:
+          timestamp_millis = timestamp.AsMicrosecondsSinceEpoch() // 1000
+          datapoints.append(_Datapoint(value=timestamp_millis,
+                                       nanos=sample.y_value))
+    return _TargetWithDatapoints(target=self._name, datapoints=datapoints)
+
+
 AVAILABLE_METRICS_LIST: List[Metric]
 AVAILABLE_METRICS_LIST = [
     ClientResourceUsageMetric("Mean User CPU Rate",
@@ -178,12 +215,21 @@ def ProcessQuery(
     (lambda n_days: f"Client Version Strings - {n_days} Day Active",
      lambda bs: data_store.REL_DB.CountClientVersionStringsByLabel(bs))
 ]
+last_active_statistics_name_fn = (
+  lambda n_days: f"Client Last Active - {n_days} days active"
+)
 # pylint: enable=unnecessary-lambda
 for metric_name_fn, metric_extract_fn in client_statistics_names_fns:
   for n_days in _FLEET_BREAKDOWN_DAY_BUCKETS:
     AVAILABLE_METRICS_LIST.append(
         ClientsStatisticsMetric(
             metric_name_fn(n_days), metric_extract_fn, n_days))
+for n_days in _LAST_ACTIVE_DAY_BUCKETS:
+  AVAILABLE_METRICS_LIST.append(
+    LastActiveMetric(
+        last_active_statistics_name_fn(n_days), n_days
+      )
+  )
 AVAILABLE_METRICS_BY_NAME = {
     metric.name: metric for metric in AVAILABLE_METRICS_LIST
 }

grr/server/grr_response_server/bin/grrafana_test.py

@@ -10,6 +10,8 @@
 
 from google.protobuf import timestamp_pb2
 
+from grr_response_core.lib import rdfvalue
+from grr_response_core.lib.rdfvalues import stats as rdf_stats
 from grr_response_server import data_store
 from grr_response_server import fleetspeak_connector
 from grr_response_server.bin import grrafana
@@ -139,6 +141,21 @@
             "foo-os": 3
         }
     })
+_TEST_LAST_ACTIVE_STATS = {
+    rdfvalue.RDFDatetime.FromHumanReadable('2020-11-25 12:29:49'): rdf_stats.ClientGraphSeries(
+        graphs=[rdf_stats.Graph(
+            data=[rdf_stats.Sample(x_value=86400000000, y_value=1),
+                  rdf_stats.Sample(x_value=172800000000, y_value=1),
+                  rdf_stats.Sample(x_value=259200000000, y_value=1),
+                  rdf_stats.Sample(x_value=604800000000, y_value=1),
+                  rdf_stats.Sample(x_value=1209600000000, y_value=1),
+                  rdf_stats.Sample(x_value=2592000000000, y_value=1),
+                  rdf_stats.Sample(x_value=5184000000000, y_value=1),
+            ]
+        )],
+        report_type=rdf_stats.ClientGraphSeries.ReportType.N_DAY_ACTIVE
+    )
+}
 _TEST_VALID_RUD_QUERY = {
     "app": "dashboard",
     "requestId": "Q119",
@@ -234,6 +251,51 @@
     "adhocFilters": [],
     "endTime": 1603276176858
 }
+_TEST_VALID_LAST_ACTIVE_QUERY = {
+  "app": "dashboard",
+  "requestId": "Q43206",
+  "timezone": "browser",
+  "panelId": 23763571993,
+  "dashboardId": 1,
+  "range": {
+    "from": "2020-10-02T13:36:28.519Z",
+    "to": "2020-12-01T13:36:28.519Z",
+    "raw": {
+      "from": "now-60d",
+      "to": "now"
+    }
+  },
+  "timeInfo": "",
+  "interval": "1h",
+  "intervalMs": 3600000,
+  "targets": [
+    {
+      "refId": "A",
+      "data": "",
+      "target": "Client Last Active - 3 days active",
+      "type": "timeseries",
+      "datasource": "grrafana"
+    }
+  ],
+  "maxDataPoints": 1825,
+  "scopedVars": {
+   "__interval": {
+     "text": "1h",
+     "value": "1h"
+    },
+   "__interval_ms": {
+      "text": "3600000",
+      "value": 3600000
+    }
+  },
+  "startTime": 1606829788519,
+  "rangeRaw": {
+    "from": "now-60d",
+    "to": "now"
+  },
+  "adhocFilters": [],
+  "endTime": 1606829788548
+}
 _TEST_INVALID_TARGET_QUERY = copy.deepcopy(_TEST_VALID_RUD_QUERY)
 _TEST_INVALID_TARGET_QUERY["targets"][0]["target"] = "unavailable_metric"
 
@@ -267,6 +329,12 @@ def _MockDatastoreReturningPlatformFleetStats(client_fleet_stats):
   return rel_db
 
 
+def _MockDatastoreReturningAllGraphSeries(graph_series_with_timestamps):
+  rel_db = mock.MagicMock()
+  rel_db.ReadAllClientGraphSeries.return_value = graph_series_with_timestamps
+  return rel_db
+
+
 class GrrafanaTest(absltest.TestCase):
   """Test the GRRafana HTTP server."""
 
@@ -304,6 +372,10 @@ def testSearchMetrics(self):
         f"Client Version Strings - {n_days} Day Active"
         for n_days in grrafana._FLEET_BREAKDOWN_DAY_BUCKETS
     ])
+    expected_res.extend([
+        f"Client Last Active - {n_days} days active"
+        for n_days in grrafana._LAST_ACTIVE_DAY_BUCKETS
+    ])
     self.assertListEqual(response.json, expected_res)
 
   def testClientResourceUsageMetricQuery(self):
@@ -355,6 +427,19 @@ def testClientsStatisticsMetric(self):
       }]
       self.assertEqual(valid_response.json, expected_res)
 
+  def testLastActiveMetric(self):
+    rel_db = _MockDatastoreReturningAllGraphSeries(
+      _TEST_LAST_ACTIVE_STATS)
+    with mock.patch.object(data_store, "REL_DB", rel_db):
+      valid_response = self.client.post(
+          "/query", json=_TEST_VALID_LAST_ACTIVE_QUERY)
+      self.assertEqual(200, valid_response.status_code)
+      expected_res = [
+         {"target": "Client Last Active - 3 days active",
+          "datapoints": [[1, 1606307389000]]}
+      ]
+      self.assertEqual(valid_response.json, expected_res)
+
 
 class TimeToProtoTimestampTest(absltest.TestCase):
   """Tests the conversion between Grafana and proto timestamps."""