fixing Complex types in Labels for Logs Details visualization

docker/clickhouse/init_schema.sql

@@ -33,6 +33,37 @@ INSERT INTO default.test_logs(event_time, content, level, id, label, detected_fi
 INSERT INTO default.test_logs(event_time, content, level, id, label, detected_field) SELECT toDateTime(now()+(number*10)) AS event_time, concat('Info Log line ', toString(number)) as content, 'Info' AS level, generateUUIDv4() as id, if(rand() % 2 = 1,'abc','cba') AS label, 1000000000.05 AS detected_field FROM numbers(1000);
 INSERT INTO default.test_logs(event_time, content, level, id, label, detected_field) SELECT toDateTime(now()+((500+number)*10)) AS event_time, concat('Unknown Log line ', toString(number)) as content, 'Unknown' AS level, generateUUIDv4() as id, if(rand() % 2 = 1,'abc','cba') AS label, 1000000000.05 AS detected_field FROM numbers(1000);
 
+DROP TABLE IF EXISTS default.test_logs_with_complex_labels;
+CREATE TABLE default.test_logs_with_complex_labels(
+  `_raw` String CODEC(ZSTD(1)),
+  `_time` DateTime64(3, 'Asia/Yekaterinburg') CODEC(ZSTD(1)),
+  `_map` Map(String, String),
+  `_db_time` DateTime DEFAULT now() CODEC(ZSTD(1)),
+  `_time_dec` Float64 DEFAULT toFloat64(_time) CODEC(DoubleDelta, Default),
+  `cluster_name` LowCardinality(String) DEFAULT JSONExtractString(_raw, 'cluster_name') CODEC(ZSTD(1)),
+  `host` LowCardinality(String) DEFAULT JSONExtractString(_raw, 'host') CODEC(ZSTD(1)),
+  `pod_namespace` LowCardinality(String) DEFAULT JSONExtractString(_raw, 'pod_namespace') CODEC(ZSTD(1)),
+  `pod_name` String DEFAULT JSONExtractString(_raw, 'pod_name') CODEC(ZSTD(1)),
+  `container_name` String DEFAULT JSONExtractString(_raw, 'container_name') CODEC(ZSTD(1)),
+  `container_image` String DEFAULT JSONExtractString(_raw, 'container_image') CODEC(ZSTD(1)),
+  `stream` LowCardinality(String) DEFAULT JSONExtractString(_raw, 'stream') CODEC(ZSTD(1)),
+  `source` LowCardinality(String) DEFAULT JSONExtractString(_raw, 'source') CODEC(ZSTD(1)),
+  `sourcetype` String DEFAULT JSONExtractString(_raw, 'source_type') CODEC(ZSTD(1)),
+  `message` String DEFAULT JSONExtractString(_raw, 'message') CODEC(ZSTD(1)),
+  `bu` LowCardinality(String) DEFAULT JSON_VALUE(_raw, '$.namespace_labels."business-unit-code"'),
+  INDEX message_ngram_bf message TYPE ngrambf_v1(4, 1024, 2, 0) GRANULARITY 1,
+  INDEX pod_name_token_bf pod_name TYPE tokenbf_v1(2048, 4, 0) GRANULARITY 4
+)
+ENGINE = MergeTree
+PARTITION BY toDate(_time)
+ORDER BY (cluster_name, bu, pod_namespace, pod_name, container_name, _time);
+
+INSERT INTO default.test_logs_with_complex_labels(_raw, _time, _map)
+SELECT '{"cluster_name":"test' || toString(number) || '","host":"test","pod_namespace":"test","pod_name":"test","container_name":"test' || toString(number) || '","container_image":"test","stream":"test","source":"test","source_type":"test","namespace_labels":{"business-unit-code":"test"}}' AS _raw,
+       now64() - INTERVAL number SECOND _time,
+       map('map_key' || toString(number),'map_value' ||toString(number)) AS _map
+FROM numbers(100);
+
 DROP TABLE IF EXISTS default.test_alerts;
 CREATE TABLE IF NOT EXISTS default.test_alerts
 (

docker/grafana/dashboards/test_logs_support.json

@@ -21,21 +21,90 @@
   "editable": true,
   "fiscalYearStartMonth": 0,
   "graphTooltip": 0,
-  "id": 6,
+  "id": 39,
   "links": [],
-  "liveNow": false,
   "panels": [
+    {
+      "datasource": {
+        "type": "vertamedia-clickhouse-datasource",
+        "uid": "P7E099F39B84EA795"
+      },
+      "description": "reproduce https://github.com/Altinity/clickhouse-grafana/issues/672",
+      "fieldConfig": {
+        "defaults": {},
+        "overrides": []
+      },
+      "gridPos": {
+        "h": 8,
+        "w": 22,
+        "x": 0,
+        "y": 0
+      },
+      "id": 4,
+      "options": {
+        "dedupStrategy": "none",
+        "enableLogDetails": true,
+        "prettifyLogMessage": true,
+        "showCommonLabels": true,
+        "showLabels": true,
+        "showTime": true,
+        "sortOrder": "Descending",
+        "wrapLogMessage": true
+      },
+      "pluginVersion": "11.3.1",
+      "targets": [
+        {
+          "adHocFilters": [
+            {
+              "key": "default.test_logs.level",
+              "operator": "=",
+              "value": "Info"
+            }
+          ],
+          "adHocValuesQuery": "",
+          "add_metadata": true,
+          "contextWindowSize": "100",
+          "database": "default",
+          "datasource": {
+            "type": "vertamedia-clickhouse-datasource",
+            "uid": "P7E099F39B84EA795"
+          },
+          "dateTimeColDataType": "_time",
+          "dateTimeType": "DATETIME64",
+          "editorMode": "sql",
+          "extrapolate": true,
+          "format": "logs",
+          "formattedQuery": "SELECT $timeSeries as t, count() FROM $table WHERE $timeFilter GROUP BY t ORDER BY t",
+          "initialized": true,
+          "interval": "",
+          "intervalFactor": 1,
+          "query": "SELECT * FROM $table WHERE $timeFilter",
+          "rawQuery": " /* grafana dashboard=Test Logs support, user=0 */\n\nSELECT *\n\nFROM default.test_logs_with_complex_labels\n\nWHERE \"_time\" >= toDateTime64(1732352032, 3) AND \"_time\" <= toDateTime64(1732524832, 3)\n",
+          "refId": "A",
+          "round": "0s",
+          "showFormattedSQL": true,
+          "skip_comments": true,
+          "table": "test_logs_with_complex_labels"
+        }
+      ],
+      "title": "Logs WIth Map as Label",
+      "type": "logs"
+    },
     {
       "datasource": {
         "type": "vertamedia-clickhouse-datasource",
         "uid": "P7E099F39B84EA795"
       },
       "description": "reproduce https://github.com/Altinity/clickhouse-grafana/issues/125 and https://github.com/Altinity/clickhouse-grafana/issues/331",
+      "fieldConfig": {
+        "defaults": {},
+        "overrides": []
+      },
       "gridPos": {
         "h": 9,
         "w": 11,
         "x": 0,
-        "y": 0
+        "y": 8
       },
       "id": 2,
       "options": {
@@ -48,8 +117,19 @@
         "sortOrder": "Descending",
         "wrapLogMessage": false
       },
+      "pluginVersion": "11.3.1",
       "targets": [
         {
+          "adHocFilters": [
+            {
+              "key": "default.test_logs.level",
+              "operator": "=",
+              "value": "Info"
+            }
+          ],
+          "adHocValuesQuery": "",
+          "add_metadata": true,
+          "contextWindowSize": "20",
           "database": "default",
           "datasource": {
             "type": "vertamedia-clickhouse-datasource",
@@ -60,15 +140,18 @@
           "dateTimeColDataType": "event_time",
           "dateTimeType": "DATETIME",
           "datetimeLoading": false,
+          "editorMode": "sql",
           "extrapolate": true,
           "format": "logs",
           "formattedQuery": "SELECT $timeSeries as t, count() FROM $table WHERE $timeFilter GROUP BY t ORDER BY t",
+          "initialized": true,
           "interval": "",
           "intervalFactor": 1,
           "query": "SELECT *\nFROM $table\n\nWHERE $timeFilter AND $adhoc\n$conditionalTest(AND content ILIKE ${filter:sqlstring},$filter)",
-          "rawQuery": "SELECT *\nFROM default.test_logs\n\nWHERE event_time >= toDateTime(1653997717) AND event_time <= toDateTime(1654019317) AND 1\n AND content ILIKE '%Info%line 11%' ",
+          "rawQuery": " /* grafana dashboard=Test Logs support, user=0 */\n\nSELECT *\n\nFROM default.test_logs\n\nWHERE\n    event_time >= toDateTime(1732347051) AND event_time <= toDateTime(1732519851)\n    AND (level = 'Info')",
           "refId": "A",
           "round": "0s",
+          "showFormattedSQL": true,
           "skip_comments": true,
           "table": "test_logs",
           "tableLoading": false
@@ -83,11 +166,15 @@
         "uid": "P4F4839B759FB0509"
       },
       "description": "",
+      "fieldConfig": {
+        "defaults": {},
+        "overrides": []
+      },
       "gridPos": {
         "h": 9,
         "w": 11,
         "x": 11,
-        "y": 0
+        "y": 8
       },
       "id": 3,
       "options": {
@@ -100,6 +187,7 @@
         "sortOrder": "Ascending",
         "wrapLogMessage": true
       },
+      "pluginVersion": "11.3.1",
       "targets": [
         {
           "builderOptions": {
@@ -192,11 +280,13 @@
       "type": "logs"
     }
   ],
-  "schemaVersion": 39,
+  "preload": false,
+  "schemaVersion": 40,
   "tags": [],
   "templating": {
     "list": [
       {
+        "baseFilters": [],
         "datasource": {
           "type": "vertamedia-clickhouse-datasource",
           "uid": "P7E099F39B84EA795"
@@ -208,18 +298,14 @@
             "value": "Info"
           }
         ],
-        "hide": 0,
         "name": "adhoc_variable",
-        "skipUrlSync": false,
         "type": "adhoc"
       },
       {
         "current": {
-          "selected": false,
           "text": "",
           "value": ""
         },
-        "hide": 0,
         "name": "filter",
         "options": [
           {
@@ -229,7 +315,6 @@
           }
         ],
         "query": "",
-        "skipUrlSync": false,
         "type": "textbox"
       }
     ]
@@ -238,9 +323,8 @@
     "from": "now-2d",
     "to": "now"
   },
-  "timeRangeUpdatedDuringEditOrView": false,
   "timepicker": {},
-  "timezone": "",
+  "timezone": "utc",
   "title": "Test Logs support",
   "uid": "VtsMXQl7z",
   "version": 1,

src/datasource/datasource.ts

@@ -74,6 +74,7 @@ export class CHDataSource
           defaultDateDate32: instanceSettings.jsonData.defaultDateDate32,
         },
         defaultDateTimeType: instanceSettings.jsonData.defaultDateTimeType,
+        contextWindowSize: instanceSettings.jsonData.contextWindowSize,
       };
     }
 
@@ -186,7 +187,7 @@ export class CHDataSource
       } PRECEDING AND CURRENT ROW) AS timestamp
           FROM $table
           ORDER BY ${inputTimestampColumn}
-        ) WHERE ${inputTimestampColumn} = '${inputTimestampValue}'`;
+        ) WHERE ${inputTimestampColumn} = ${inputTimestampValue}`;
     };
 
     const generateQueryForTimestampForward = (inputTimestampColumn, inputTimestampValue, contextWindowSize) => {
@@ -198,7 +199,7 @@ export class CHDataSource
       } FOLLOWING) AS timestamp
           FROM $table
           ORDER BY ${inputTimestampColumn}
-        ) WHERE ${inputTimestampColumn} = '${inputTimestampValue}'`;
+        ) WHERE ${inputTimestampColumn} = ${inputTimestampValue}`;
     };
 
     const generateRequestForTimestampForward = (timestampField, timestamp, currentRowTimestamp, select) => {
@@ -236,19 +237,26 @@ export class CHDataSource
       const timestampColumn = query?.dateTimeColDataType;
 
       const getLogsTimeBoundaries = async () => {
+        let formattedDate = String(row.timeEpochMs);
+        if (formattedDate.length > 10) {
+          formattedDate = `toDateTime64(${row.timeEpochMs}/1000,3)`;
+        } else {
+          formattedDate = `'${row.timeUtc}'`;
+        }
+
         const boundariesRequest =
           options?.direction === LogRowContextQueryDirection.Backward
-            ? generateQueryForTimestampBackward(timestampColumn, row.timeUtc, query?.contextWindowSize)
-            : generateQueryForTimestampForward(timestampColumn, row.timeUtc, query?.contextWindowSize);
+            ? generateQueryForTimestampBackward(timestampColumn, formattedDate, query?.contextWindowSize)
+            : generateQueryForTimestampForward(timestampColumn, formattedDate, query?.contextWindowSize);
 
         const { stmt, requestId } = this.createQuery(requestOptions, { ...query, query: boundariesRequest });
 
         const result: any = await this._seriesQuery(stmt, requestId + options?.direction);
+
         return result.data[0];
       };
 
       const { timestamp } = await getLogsTimeBoundaries();
-
       const getLogContext = async () => {
         const contextDataRequest =
           options?.direction === LogRowContextQueryDirection.Backward

src/datasource/sql-series/toLogs.ts

@@ -86,7 +86,33 @@ export const toLogs = (self: any): DataFrame[] => {
       }
 
       if (key === messageField) {
-        frame.addField({ name: key, type: types[key], labels: labels });
+        const transformObject = (obj) => {
+          // Check if the input is an object and not null
+          if (obj && typeof obj === 'object') {
+            // Create a new object to store the transformed properties
+            const result = Array.isArray(obj) ? [] : {};
+
+            for (const key in obj) {
+              if (Object.prototype.hasOwnProperty.call(obj, key)) {
+                const value = obj[key];
+
+                // If the value is an object (and not null), convert it to a string
+                if (value && typeof value === 'object') {
+                  result[key] = JSON.stringify(value);
+                } else {
+                  // Otherwise, keep the primitive value as it is
+                  result[key] = value;
+                }
+              }
+            }
+
+            return result;
+          }
+          // Return the original value if it's not an object
+          return obj;
+        }
+
+        frame.addField({ name: key, type: types[key], labels: transformObject(labels), config: { filterable: false } });
       } else if (!labelFields.includes(key) && types[key].fieldType === FieldType.time) {
         frame.addField({ name: key, type: FieldType.time });
       } else if (!labelFields.includes(key)) {

src/types/types.ts

@@ -42,7 +42,7 @@ export interface CHQuery extends DataQuery {
   intervalFactor?: number;
   interval?: string;
   formattedQuery?: string;
-  contextWindowSize?: number;
+  contextWindowSize?: string;
   adHocValuesQuery?: string;
 }
 
@@ -66,6 +66,7 @@ export interface CHDataSourceOptions extends DataSourceJsonData {
   defaultDateDate32?: string;
   defaultDateTimeType?: string;
   adHocValuesQuery: string;
+  contextWindowSize?: string;
 }
 
 /**

src/views/ConfigEditor/ConfigEditor.tsx

@@ -6,8 +6,8 @@ import _ from 'lodash';
 import { DefaultValues } from './FormParts/DefaultValues/DefaultValues';
 import { LANGUAGE_ID } from '../QueryEditor/components/QueryTextEditor/editor/initiateEditor';
 import { MONACO_EDITOR_OPTIONS } from '../constants';
-import {COMPRESSION_TYPE_OPTIONS} from "./constants";
-import {DEFAULT_VALUES_QUERY} from "../../datasource/adhoc";
+import { COMPRESSION_TYPE_OPTIONS } from './constants';
+import { DEFAULT_VALUES_QUERY } from '../../datasource/adhoc';
 
 export interface CHSecureJsonData {
   password?: string;