Add logs tag attributes v2 with max distinct values limit (#477)

Logs part of #467
SigNoz · Dec 11, 2024 · aff7ae4 · aff7ae4
1 parent f848d2b
commit aff7ae4
Show file tree

Hide file tree

Showing 6 changed files with 236 additions and 35 deletions.
diff --git a/cmd/signozschemamigrator/schema_migrator/logs_migrations.go b/cmd/signozschemamigrator/schema_migrator/logs_migrations.go
@@ -80,4 +80,55 @@ ORDER BY name ASC`,
 			},
 		},
 	},
+	{
+		MigrationID: 1001,
+		UpItems: []Operation{
+			CreateTableOperation{
+				Database: "signoz_logs",
+				Table:    "tag_attributes_v2",
+				Columns: []Column{
+					{Name: "unix_milli", Type: ColumnTypeInt64, Codec: "Delta(8), ZSTD(1)"},
+					{Name: "tag_key", Type: ColumnTypeString, Codec: "ZSTD(1)"},
+					{Name: "tag_type", Type: LowCardinalityColumnType{ColumnTypeString}, Codec: "ZSTD(1)"},
+					{Name: "tag_data_type", Type: LowCardinalityColumnType{ColumnTypeString}, Codec: "ZSTD(1)"},
+					{Name: "string_value", Type: ColumnTypeString, Codec: "ZSTD(1)"},
+					{Name: "number_value", Type: NullableColumnType{ColumnTypeFloat64}, Codec: "ZSTD(1)"},
+				},
+				Indexes: []Index{
+					{Name: "string_value_index", Expression: "string_value", Type: "ngrambf_v1(4, 1024, 3, 0)", Granularity: 1},
+					{Name: "number_value_index", Expression: "number_value", Type: "minmax", Granularity: 1},
+				},
+				Engine: ReplacingMergeTree{
+					MergeTree: MergeTree{
+						PartitionBy: "toDate(unix_milli / 1000)",
+						OrderBy:     "(tag_key, tag_type, tag_data_type, string_value, number_value)",
+						TTL:         "toDateTime(unix_milli / 1000) + toIntervalSecond(1296000)",
+						Settings: TableSettings{
+							{Name: "index_granularity", Value: "8192"},
+							{Name: "ttl_only_drop_parts", Value: "1"},
+							{Name: "allow_nullable_key", Value: "1"},
+						},
+					},
+				},
+			},
+			CreateTableOperation{
+				Database: "signoz_logs",
+				Table:    "distributed_tag_attributes_v2",
+				Columns: []Column{
+					{Name: "unix_milli", Type: ColumnTypeInt64, Codec: "Delta(8), ZSTD(1)"},
+					{Name: "tag_key", Type: ColumnTypeString, Codec: "ZSTD(1)"},
+					{Name: "tag_type", Type: LowCardinalityColumnType{ColumnTypeString}, Codec: "ZSTD(1)"},
+					{Name: "tag_data_type", Type: LowCardinalityColumnType{ColumnTypeString}, Codec: "ZSTD(1)"},
+					{Name: "string_value", Type: ColumnTypeString, Codec: "ZSTD(1)"},
+					{Name: "number_value", Type: NullableColumnType{ColumnTypeFloat64}, Codec: "ZSTD(1)"},
+				},
+				Engine: Distributed{
+					Database:    "signoz_logs",
+					Table:       "tag_attributes_v2",
+					ShardingKey: "cityHash64(rand())",
+				},
+			},
+		},
+		DownItems: []Operation{},
+	},
 }
diff --git a/exporter/clickhouselogsexporter/config.go b/exporter/clickhouselogsexporter/config.go
@@ -16,12 +16,18 @@ package clickhouselogsexporter
 
 import (
 	"errors"
+	"time"
 
 	"go.opentelemetry.io/collector/config/configretry"
 	"go.opentelemetry.io/collector/exporter/exporterhelper"
 	"go.uber.org/multierr"
 )
 
+type AttributesLimits struct {
+	FetchKeysInterval time.Duration `mapstructure:"fetch_keys_interval" default:"10m"`
+	MaxDistinctValues int           `mapstructure:"max_distinct_values" default:"25000"`
+}
+
 // Config defines configuration for ClickHouse exporter.
 type Config struct {
 	exporterhelper.TimeoutConfig `mapstructure:",squash"`
@@ -33,6 +39,8 @@ type Config struct {
 	// For http protocol reference: [mailru/go-clickhouse/#dsn](https://github.com/mailru/go-clickhouse/#dsn).
 	DSN          string `mapstructure:"dsn"`
 	UseNewSchema bool   `mapstructure:"use_new_schema" default:"false"`
+
+	AttributesLimits AttributesLimits `mapstructure:"attributes_limits"`
 }
 
 var (

diff --git a/exporter/clickhouselogsexporter/config_test.go b/exporter/clickhouselogsexporter/config_test.go
@@ -63,6 +63,10 @@ func TestLoadConfig(t *testing.T) {
 			NumConsumers: 10,
 			QueueSize:    100,
 		},
+		AttributesLimits: AttributesLimits{
+			FetchKeysInterval: 10 * time.Minute,
+			MaxDistinctValues: 25000,
+		},
 	})
 
 	defaultCfg.(*Config).UseNewSchema = true