feat: support function-format parameters in Insight Data API (#93)

* feat: support function-format parameters in Insight Data API

* chore: update license header

* chore: format code

* feat: return `complex_fields` in the view API

* feat: include system cluster rollup setting in App Settings API response

* chore: update release notes

* fix: remove func `sum_func_value_in_group` from using aggregation `date_histogram`

* fix: generate unique group id for aggregation name

* fix: sort not work when using function `sum_func_value_in_group`

* feat: update view template

docs/content.en/docs/release-notes/_index.md

@@ -11,6 +11,7 @@ Information about release notes of INFINI Console is provided here.
 
 ### Breaking changes
 ### Features
+- Support function-format parameters in Insight Data API
 - Support configuring multiple hosts when creating a cluster
 ### Bug fix
 ### Improvements

docs/content.zh/docs/release-notes/_index.md

@@ -7,6 +7,14 @@ title: "版本历史"
 
 这里是 INFINI Console 历史版本发布的相关说明。
 
+## Latest (In development)
+
+### Breaking changes
+### Features
+- Insight Data API 支持函数格式查询，方便拓展查询功能
+- 创建集群时支持配置多个主机地址，增强集群的高可用性
+### Bug fix
+### Improvements
 
 ## 1.28.0 (2025-01-11)
 

main.go

@@ -33,6 +33,7 @@ import (
 	"infini.sh/framework/core/api"
 	"infini.sh/framework/core/host"
 	model2 "infini.sh/framework/core/model"
+	"infini.sh/framework/core/util"
 	elastic2 "infini.sh/framework/modules/elastic"
 	_ "time/tzdata"
 
@@ -185,6 +186,29 @@ func main() {
 				}
 				return err
 			})
+			api.RegisterAppSetting("system_cluster", func() interface{} {
+				client := elastic.GetClient(global.MustLookupString(elastic.GlobalSystemElasticsearchID))
+				ver := client.GetVersion()
+				if ver.Distribution != elastic.Easysearch {
+					return map[string]interface{}{
+						"rollup_enabled": false,
+					}
+				}
+				settings, err := client.GetClusterSettings(nil)
+				if err != nil {
+					log.Errorf("failed to get cluster settings with system cluster: %v", err)
+					return nil
+				}
+
+				rollupEnabled, _ := util.GetMapValueByKeys([]string{"persistent", "rollup", "search", "enabled"}, settings)
+				rollupEnabledValue := false
+				if v, ok := rollupEnabled.(string); ok && v == "true" {
+					rollupEnabledValue = true
+				}
+				return map[string]interface{}{
+					"rollup_enabled": rollupEnabledValue,
+				}
+			})
 		}
 
 		if !global.Env().SetupRequired() {

model/insight/metric_data.go

@@ -35,6 +35,29 @@ import (
 	"infini.sh/framework/core/util"
 )
 
+const (
+	AggFuncCount                      = "count"
+	AggFuncAvg                        = "avg"
+	AggFuncSum                        = "sum"
+	AggFuncMin                        = "min"
+	AggFuncMax                        = "max"
+	AggFuncMedium                     = "medium"
+	AggFuncValueCount                 = "value_count"
+	AggFuncCardinality                = "cardinality"
+	AggFuncDerivative                 = "derivative"
+	AggFuncRate                       = "rate"
+	AggFuncPercent99                  = "p99"
+	AggFuncPercent95                  = "p95"
+	AggFuncPercent90                  = "p90"
+	AggFuncPercent80                  = "p80"
+	AggFuncPercent50                  = "p50"
+	AggFuncLatest                     = "latest"
+	AggFuncLatency                    = "latency"
+	AggFuncSumFuncValueInGroup        = "sum_func_value_in_group"
+	AggFuncRateSumFuncValueInGroup    = "rate_sum_func_value_in_group"
+	AggFuncLatencySumFuncValueInGroup = "latency_sum_func_value_in_group"
+)
+
 type Metric struct {
 	AggTypes     []string          `json:"agg_types,omitempty"`
 	IndexPattern string            `json:"index_pattern,omitempty"`
@@ -108,14 +131,55 @@ func (m *Metric) GenerateExpression() (string, error) {
 
 	return string(expressionBytes), nil
 }
-func (m *Metric) AutoTimeBeforeGroup() bool {
+
+// shouldUseAggregation checks if any item's statistic or function exists in the provided aggFuncs list.
+// If a match is found, it returns true; otherwise, it returns false.
+func (m *Metric) shouldUseAggregation(aggFuncs []string) bool {
 	for _, item := range m.Items {
-		if item.Statistic == "derivative" {
-			return false
+		// Default to item's Statistic field
+		statistic := item.Statistic
+
+		// If Function is defined, use its first key as the statistic
+		if item.Function != nil {
+			for key := range item.Function {
+				statistic = key
+				break
+			}
+		}
+
+		// Check if statistic is in the aggregation function list
+		if util.StringInArray(aggFuncs, statistic) {
+			return true
 		}
 	}
-	return true
+	return false
 }
+
+// AutoTimeBeforeGroup determines if date aggregation should be applied before terms aggregation.
+// Returns false if the metric uses any of the specified aggregation functions.
+func (m *Metric) AutoTimeBeforeGroup() bool {
+	return !m.shouldUseAggregation([]string{
+		AggFuncDerivative,
+		AggFuncRate,
+		AggFuncLatency,
+		AggFuncRateSumFuncValueInGroup,
+		AggFuncLatencySumFuncValueInGroup,
+	})
+}
+
+// UseBucketSort determines whether bucket sorting should be used for aggregation.
+// Returns false if the metric contains specific aggregation functions that require alternative handling.
+func (m *Metric) UseBucketSort() bool {
+	return m.shouldUseAggregation([]string{
+		AggFuncDerivative,
+		AggFuncRate,
+		AggFuncLatency,
+		AggFuncSumFuncValueInGroup,
+		AggFuncRateSumFuncValueInGroup,
+		AggFuncLatencySumFuncValueInGroup,
+	})
+}
+
 func (m *Metric) ValidateSortKey() error {
 	if len(m.Sort) == 0 {
 		return nil
@@ -143,6 +207,10 @@ type MetricItem struct {
 	Field     string `json:"field"`
 	FieldType string `json:"field_type,omitempty"`
 	Statistic string `json:"statistic,omitempty"`
+
+	//Function specifies the calculation details for the metric,
+	//including the aggregation type and any associated parameters.
+	Function map[string]interface{} `json:"function,omitempty"`
 }
 
 type MetricDataItem struct {

modules/elastic/api/view.go

@@ -128,10 +128,11 @@ func (h *APIHandler) HandleGetViewListAction(w http.ResponseWriter, req *http.Re
 				"title":    hit.Source["title"],
 				"viewName": hit.Source["viewName"],
 			},
-			"score":      0,
-			"type":       "index-pattern",
-			"namespaces": []string{"default"},
-			"updated_at": hit.Source["updated_at"],
+			"score":          0,
+			"type":           "index-pattern",
+			"namespaces":     []string{"default"},
+			"updated_at":     hit.Source["updated_at"],
+			"complex_fields": hit.Source["complex_fields"],
 		}
 		savedObjects = append(savedObjects, savedObject)
 	}
@@ -329,6 +330,8 @@ func (h *APIHandler) HandleBulkGetViewAction(w http.ResponseWriter, req *http.Re
 			"namespaces":       []string{"default"},
 			"migrationVersion": map[string]interface{}{"index-pattern": "7.6.0"},
 			"updated_at":       hit.Source["updated_at"],
+			"complex_fields":   hit.Source["complex_fields"],
+			"references":       []interface{}{},
 		}
 		savedObjects = append(savedObjects, savedObject)
 	}

plugin/api/insight/function.go

@@ -0,0 +1,35 @@
+// Copyright (C) INFINI Labs & INFINI LIMITED.
+//
+// The INFINI Console is offered under the GNU Affero General Public License v3.0
+// and as commercial software.
+//
+// For commercial licensing, contact us at:
+//   - Website: infinilabs.com
+//   - Email: [email protected]
+//
+// Open Source licensed under AGPL V3:
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+// GNU Affero General Public License for more details.
+//
+// You should have received a copy of the GNU Affero General Public License
+// along with this program. If not, see <http://www.gnu.org/licenses/>.
+
+/* Copyright © INFINI Ltd. All rights reserved.
+ * Web: https://infinilabs.com
+ * Email: hello#infini.ltd */
+
+package insight
+
+type Function interface {
+	//GenerateAggregation generates aggregation for specify function, e.g. rate, latency ...
+	//
+	// metricName: The name of the metric to be calculated (used as an identifier in the aggregation).
+	GenerateAggregation(metricName string) (map[string]interface{}, error)
+}

plugin/api/insight/function/latency.go

@@ -0,0 +1,86 @@
+// Copyright (C) INFINI Labs & INFINI LIMITED.
+//
+// The INFINI Console is offered under the GNU Affero General Public License v3.0
+// and as commercial software.
+//
+// For commercial licensing, contact us at:
+//   - Website: infinilabs.com
+//   - Email: [email protected]
+//
+// Open Source licensed under AGPL V3:
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+// GNU Affero General Public License for more details.
+//
+// You should have received a copy of the GNU Affero General Public License
+// along with this program. If not, see <http://www.gnu.org/licenses/>.
+
+/* Copyright © INFINI Ltd. All rights reserved.
+ * Web: https://infinilabs.com
+ * Email: hello#infini.ltd */
+
+package function
+
+import (
+	"fmt"
+	"infini.sh/framework/core/util"
+)
+
+// Latency represents a function that calculates latency as a ratio of two metrics.
+// Divisor: The denominator in the latency calculation.
+// Dividend: The numerator in the latency calculation.
+type Latency struct {
+	Divisor  string `json:"divisor"`
+	Dividend string `json:"dividend"`
+}
+
+// GenerateAggregation implements the `Function` interface and generates an aggregation for the `latency` function.
+func (p *Latency) GenerateAggregation(metricName string) (map[string]interface{}, error) {
+	if p.Dividend == "" || p.Divisor == "" {
+		return nil, fmt.Errorf("empty divisor or dividend for agg func: latency")
+	}
+
+	var (
+		divisorAggID      = util.GetUUID()
+		dividendAggID     = util.GetUUID()
+		divisorBaseAggID  = util.GetUUID()
+		dividendBaseAggID = util.GetUUID()
+	)
+	return util.MapStr{
+		dividendBaseAggID: util.MapStr{
+			"max": util.MapStr{
+				"field": p.Dividend,
+			},
+		},
+		dividendAggID: util.MapStr{
+			"derivative": util.MapStr{
+				"buckets_path": dividendBaseAggID,
+			},
+		},
+		divisorBaseAggID: util.MapStr{
+			"max": util.MapStr{
+				"field": p.Divisor,
+			},
+		},
+		divisorAggID: util.MapStr{
+			"derivative": util.MapStr{
+				"buckets_path": divisorBaseAggID,
+			},
+		},
+		metricName: util.MapStr{
+			"bucket_script": util.MapStr{
+				"buckets_path": util.MapStr{
+					"dividend": dividendAggID,
+					"divisor":  divisorAggID,
+				},
+				"script": "params.dividend / params.divisor",
+			},
+		},
+	}, nil
+}