From 41a28e57bea785b7f6f82153abf8629cd47d3f22 Mon Sep 17 00:00:00 2001 From: Kyle <1023070+kyle-sammons@users.noreply.github.com> Date: Thu, 24 Oct 2024 13:30:40 -0700 Subject: [PATCH] Swap to using integers for extended bounds in the data histogram aggregation (#61) * Swap to using integers for extended bounds in the data histogram aggregation instead of strings so that alerts using _timesinceepoch continue to work * Update docs * Fixed tests --- .github/CONTRIBUTING.md | 9 ++++----- docker-compose.yaml | 3 +++ go.mod | 8 +++++++- pkg/astra/astra.go | 2 ++ pkg/astra/client/client.go | 3 +-- pkg/astra/client/models.go | 6 +++--- pkg/astra/lucene_handler.go | 4 ++-- pkg/astra/time_series_query_test.go | 14 ++++++++------ 8 files changed, 30 insertions(+), 19 deletions(-) diff --git a/.github/CONTRIBUTING.md b/.github/CONTRIBUTING.md index 6d1ae88..fc1e680 100644 --- a/.github/CONTRIBUTING.md +++ b/.github/CONTRIBUTING.md @@ -119,11 +119,10 @@ Astra datasource URL inside your Grafana instance. mage -v ``` -3. List all available Mage targets for additional commands: - - ```bash - mage -l - ``` +3. Move the binaries into the target directory +```bash +mv dist/gpx_slack_astra_app_datasource_backend_* dist/datasource +``` ### Releases diff --git a/docker-compose.yaml b/docker-compose.yaml index f1de6a6..6660b77 100644 --- a/docker-compose.yaml +++ b/docker-compose.yaml @@ -21,3 +21,6 @@ services: GF_PATHS_PLUGINS: "/var/lib/grafana/plugins" GF_PLUGINS_ALLOW_LOADING_UNSIGNED_PLUGINS: "slack-astra-app,slack-astra-app-backend-datasource" GF_SERVER_ENABLE_GZIP: "true" + GF_UNIFIED_ALERTING_ENABLED: "false" + GF_ALERTING_ENABLED: "true" + GF_DATABASE_WAL: "true" diff --git a/go.mod b/go.mod index 7ee2076..8579e8b 100644 --- a/go.mod +++ b/go.mod @@ -12,6 +12,12 @@ require ( golang.org/x/net v0.0.0-20210614182718-04defd469f4e ) -require github.com/grafana/opensearch-datasource v1.2.0 +require ( + github.com/golang/groupcache v0.0.0-20200121045136-8c9f03a8e57e // indirect + github.com/grafana/opensearch-datasource v1.2.0 + github.com/jung-kurt/gofpdf v1.0.3-0.20190309125859-24315acbbda5 // indirect + github.com/sirupsen/logrus v1.6.0 // indirect + golang.org/x/exp v0.0.0-20200224162631-6cc2880d07d6 // indirect +) replace golang.org/x/sys => golang.org/x/sys v0.0.0-20220615213510-4f61da869c0c diff --git a/pkg/astra/astra.go b/pkg/astra/astra.go index 10855b1..5db6802 100644 --- a/pkg/astra/astra.go +++ b/pkg/astra/astra.go @@ -16,6 +16,8 @@ type AstraExecutor struct{} var ( intervalCalculator tsdb.IntervalCalculator + _ backend.QueryDataHandler = (*AstraDatasource)(nil) + _ backend.CheckHealthHandler = (*AstraDatasource)(nil) ) type TsdbQueryEndpoint interface { diff --git a/pkg/astra/client/client.go b/pkg/astra/client/client.go index 0752319..d361b37 100644 --- a/pkg/astra/client/client.go +++ b/pkg/astra/client/client.go @@ -208,7 +208,6 @@ func (c *baseClientImpl) encodeBatchRequests(requests []*multiRequest) ([]byte, body := string(reqBody) body = strings.ReplaceAll(body, "$__interval_ms", strconv.FormatInt(r.interval.Milliseconds(), 10)) body = strings.ReplaceAll(body, "$__interval", r.interval.Text) - payload.WriteString(body + "\n") } @@ -464,7 +463,7 @@ func (c *baseClientImpl) executePPLQueryRequest(method, uriPath string, body []b return nil, err } - clientLog.Debug("Executing request", "url", req.URL.String(), "method", method) + clientLog.Debug("Executing PPL request", "url", req.URL.String(), "method", method) var reqInfo *PPLRequestInfo if c.debugEnabled { diff --git a/pkg/astra/client/models.go b/pkg/astra/client/models.go index f5c24f5..2225374 100644 --- a/pkg/astra/client/models.go +++ b/pkg/astra/client/models.go @@ -80,7 +80,7 @@ type SearchResponse struct { Error map[string]interface{} `json:"error"` Aggregations map[string]interface{} `json:"aggregations"` Hits *SearchResponseHits `json:"hits"` - Shards map[string]interface{} `json:"_shards"` + Shards map[string]interface{} `json:"_shards"` } // MultiSearchRequest represents a multi search request @@ -271,8 +271,8 @@ type TermsAggregation struct { // ExtendedBounds represents extended bounds type ExtendedBounds struct { - Min string `json:"min"` - Max string `json:"max"` + Min int64 `json:"min"` + Max int64 `json:"max"` } // GeoHashGridAggregation represents a geo hash grid aggregation diff --git a/pkg/astra/lucene_handler.go b/pkg/astra/lucene_handler.go index fd5e1db..e51a22c 100644 --- a/pkg/astra/lucene_handler.go +++ b/pkg/astra/lucene_handler.go @@ -69,7 +69,7 @@ func (h *luceneHandler) processQuery(q *Query) error { for _, bucketAgg := range q.BucketAggs { switch bucketAgg.Type { case dateHistType: - aggBuilder = addDateHistogramAgg(aggBuilder, bucketAgg, from, to) + aggBuilder = addDateHistogramAgg(aggBuilder, bucketAgg, int64(fromMs), int64(toMs)) case histogramType: aggBuilder = addHistogramAgg(aggBuilder, bucketAgg) case filtersType: @@ -168,7 +168,7 @@ func (h *luceneHandler) executeQueries() (*backend.QueryDataResponse, error) { return rp.getTimeSeries() } -func addDateHistogramAgg(aggBuilder es.AggBuilder, bucketAgg *BucketAgg, timeFrom, timeTo string) es.AggBuilder { +func addDateHistogramAgg(aggBuilder es.AggBuilder, bucketAgg *BucketAgg, timeFrom, timeTo int64) es.AggBuilder { aggBuilder.DateHistogram(bucketAgg.ID, bucketAgg.Field, func(a *es.DateHistogramAgg, b es.AggBuilder) { a.Interval = bucketAgg.Settings.Get("interval").MustString("auto") a.MinDocCount = bucketAgg.Settings.Get("min_doc_count").MustInt(0) diff --git a/pkg/astra/time_series_query_test.go b/pkg/astra/time_series_query_test.go index c03fc93..c14b338 100644 --- a/pkg/astra/time_series_query_test.go +++ b/pkg/astra/time_series_query_test.go @@ -15,8 +15,10 @@ import ( func TestExecuteTimeSeriesQuery(t *testing.T) { from := time.Date(2018, 5, 15, 17, 50, 0, 0, time.UTC) to := time.Date(2018, 5, 15, 17, 55, 0, 0, time.UTC) - fromStr := fmt.Sprintf("%d", from.UnixNano()/int64(time.Millisecond)) - toStr := fmt.Sprintf("%d", to.UnixNano()/int64(time.Millisecond)) + fromMs := from.UnixNano() / int64(time.Millisecond) + toMs := to.UnixNano() / int64(time.Millisecond) + fromStr := fmt.Sprintf("%d", fromMs) + toStr := fmt.Sprintf("%d", toMs) Convey("Test execute time series query", t, func() { Convey("With defaults on Elasticsearch 2.0.0", func() { @@ -36,8 +38,8 @@ func TestExecuteTimeSeriesQuery(t *testing.T) { So(sr.Aggs[0].Key, ShouldEqual, "2") dateHistogramAgg := sr.Aggs[0].Aggregation.Aggregation.(*es.DateHistogramAgg) So(dateHistogramAgg.Field, ShouldEqual, "@timestamp") - So(dateHistogramAgg.ExtendedBounds.Min, ShouldEqual, fromStr) - So(dateHistogramAgg.ExtendedBounds.Max, ShouldEqual, toStr) + So(dateHistogramAgg.ExtendedBounds.Min, ShouldEqual, fromMs) + So(dateHistogramAgg.ExtendedBounds.Max, ShouldEqual, toMs) }) Convey("With defaults on Elasticsearch 5.0.0", func() { @@ -51,8 +53,8 @@ func TestExecuteTimeSeriesQuery(t *testing.T) { sr := c.multisearchRequests[0].Requests[0] So(sr.Query.Bool.Filters[0].(*es.RangeFilter).Key, ShouldEqual, c.timeField) So(sr.Aggs[0].Key, ShouldEqual, "2") - So(sr.Aggs[0].Aggregation.Aggregation.(*es.DateHistogramAgg).ExtendedBounds.Min, ShouldEqual, fromStr) - So(sr.Aggs[0].Aggregation.Aggregation.(*es.DateHistogramAgg).ExtendedBounds.Max, ShouldEqual, toStr) + So(sr.Aggs[0].Aggregation.Aggregation.(*es.DateHistogramAgg).ExtendedBounds.Min, ShouldEqual, fromMs) + So(sr.Aggs[0].Aggregation.Aggregation.(*es.DateHistogramAgg).ExtendedBounds.Max, ShouldEqual, toMs) }) Convey("With multiple bucket aggs", func() {