timescale · Mar 21, 2024 · Mar 21, 2024 · Mar 22, 2024 · Mar 22, 2024 · Mar 22, 2024
Showing with 245 additions and 2 deletions.

+38 −0 .github/workflows/build-n-push.yml

+22 −0 deployment.yaml

+75 −0 pkg/publicid/publicid.go

+104 −0 pkg/publicid/publicid_test.go

+6 −2 pkg/targets/clickhouse/creator.go
diff --git a/.github/workflows/build-n-push.yml b/.github/workflows/build-n-push.yml
@@ -0,0 +1,38 @@
+name: Build and push the image to ECR
+
+on:
+    workflow_dispatch:
+        inputs:
+            tag:
+                description: 'Tag for the image'
+                required: true
+                default: 'latest'
+
+jobs:
+    build-and-push:
+        runs-on: ubuntu-22.04
+        steps:
+            - name: Checkout
+              uses: actions/checkout@v4
+              timeout-minutes: 5
+
+            - name: Configure AWS credentials
+              uses: aws-actions/configure-aws-credentials@010d0da01d0b5a38af31e9c3470dbfdabdecca3a
+              timeout-minutes: 5
+              with:
+                aws-access-key-id: ${{ secrets.AWS_ACCESS_KEY_ID }}
+                aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
+                aws-region: eu-central-1
+
+            - name: Login to Amazon ECR
+              uses: aws-actions/amazon-ecr-login@062b18b96a7aff071d4dc91bc00c4c1a7945b076
+              timeout-minutes: 5
+
+            - name: Build and push the image 
+              timeout-minutes: 5
+              env:
+                REPO_URL: 698591829921.dkr.ecr.eu-central-1.amazonaws.com/tsbs
+                IMAGE_TAG: ${{ github.event.inputs.tag }}
+              run: |
+                docker build -t $REPO_URL:$IMAGE_TAG .
+                docker push $REPO_URL:$IMAGE_TAG
diff --git a/deployment.yaml b/deployment.yaml
@@ -0,0 +1,22 @@
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: tsbs
+  namespace: default
+spec:
+  replicas: 1
+  selector:
+    matchLabels:
+      app: tsbs
+  template:
+    metadata:
+      labels:
+        app: tsbs
+    spec:
+      containers:
+      - name: tsbs
+        image: 698591829921.dkr.ecr.eu-central-1.amazonaws.com/tsbs:latest
+        resources:
+          limits:
+            memory: "1Gi"
+            cpu: "1"
diff --git a/pkg/publicid/publicid.go b/pkg/publicid/publicid.go
@@ -0,0 +1,75 @@
+package publicid
+
+import (
+	"fmt"
+	"strings"
+
+	nanoid "github.com/matoous/go-nanoid/v2"
+)
+
+const (
+	Alphabet = "123456789ABCDEFGHJKLMNPQRSTUVWXYZabcdefghijkmnopqrstuvwxyz" // base58
+	Length   = 12
+)
+
+// New generates a unique public ID.
+func New() (string, error) {
+	return nanoid.Generate(Alphabet, Length)
+}
+
+// NewWithPrefix generates a unique public ID with a given prefix.
+func NewWithPrefix(prefix string) (string, error) {
+	id, err := New()
+	if err != nil {
+		return "", err
+	}
+
+	return fmt.Sprintf("%s_%s", prefix, id), nil
+}
+
+// Must is the same as New, but panics on error.
+func Must() string {
+	return nanoid.MustGenerate(Alphabet, Length)
+}
+
+// MustWithPrefix is the same as NewWithPrefix, but panics on error.
+func MustWithPrefix(prefix string) string {
+	id := Must()
+	return fmt.Sprintf("%s_%s", prefix, id)
+}
+
+// Validate checks if a given field name’s public ID value is valid according to
+// the constraints defined by package publicid.
+func Validate(fieldName, id string) error {
+	if id == "" {
+		return fmt.Errorf("%s cannot be blank", fieldName)
+	}
+
+	if len(id) != Length {
+		return fmt.Errorf("%s should be %d characters long", fieldName, Length)
+	}
+
+	if strings.Trim(id, Alphabet) != "" {
+		return fmt.Errorf("%s has invalid characters", fieldName)
+	}
+
+	return nil
+}
+
+// ValidateWithPrefix checks if a given field name’s public ID value with a prefix is valid according to
+// the constraints defined by package publicid.
+func ValidateWithPrefix(fieldName, id, prefix string) error {
+	if id == "" {
+		return fmt.Errorf("%s cannot be blank", fieldName)
+	}
+
+	expectedPrefix := fmt.Sprintf("%s_", prefix)
+	if !strings.HasPrefix(id, expectedPrefix) {
+		return fmt.Errorf("%s must start with the prefix %s", fieldName, prefix)
+	}
+
+	// Remove the prefix and the underscore from the ID for further validation.
+	trimmedID := strings.TrimPrefix(id, expectedPrefix)
+
+	return Validate(fieldName, trimmedID)
+}
diff --git a/pkg/publicid/publicid_test.go b/pkg/publicid/publicid_test.go
@@ -0,0 +1,104 @@
+package publicid
+
+import (
+	"fmt"
+	"strings"
+	"testing"
+
+	"github.com/stretchr/testify/assert"
+)
+
+func TestNew(t *testing.T) {
+	id, err := New()
+	assert.NoError(t, err)
+	assert.Len(t, id, Length, fmt.Sprintf("ID should be %d characters long", Length))
+	assert.Equal(t, "", strings.Trim(id, Alphabet), "ID should only contain characters from the Alphabet")
+}
+
+func TestNewWithPrefix(t *testing.T) {
+	prefix := "test"
+	id, err := NewWithPrefix(prefix)
+	assert.NoError(t, err)
+	assert.True(t, strings.HasPrefix(id, prefix+"_"), "ID should have the correct prefix")
+	assert.Len(t, id, len(prefix)+1+Length, fmt.Sprintf("ID should be %d characters long", len(prefix)+1+Length))
+	assert.Equal(t, "", strings.Trim(id[len(prefix)+1:], Alphabet), "ID should only contain characters from the Alphabet after the prefix")
+}
+
+func TestMust(t *testing.T) {
+	assert.NotPanics(t, func() {
+		id := Must()
+		assert.Len(t, id, Length, fmt.Sprintf("ID should be %d characters long", Length))
+		assert.Equal(t, "", strings.Trim(id, Alphabet), "ID should only contain characters from the Alphabet")
+	})
+}
+
+func TestMustWithPrefix(t *testing.T) {
+	prefix := "must"
+	assert.NotPanics(t, func() {
+		id := MustWithPrefix(prefix)
+		assert.True(t, strings.HasPrefix(id, prefix+"_"), "ID should have the correct prefix")
+		assert.Len(t, id, len(prefix)+1+Length, fmt.Sprintf("ID should be %d characters long", len(prefix)+1+Length))
+		assert.Equal(t, "", strings.Trim(id[len(prefix)+1:], Alphabet), "ID should only contain characters from the Alphabet after the prefix")
+	})
+}
+
+func TestValidate(t *testing.T) {
+	validId, _ := New()
+
+	testCases := []struct {
+		name        string
+		fieldName   string
+		id          string
+		wantErr     bool
+		errContains string
+	}{
+		{"Valid ID", "field", validId, false, ""},
+		{"Empty ID", "field", "", true, "cannot be blank"},
+		{"Incorrect Length", "field", "123", true, "should be 12 characters long"},
+		{"Invalid Characters", "field", "invalidchar!", true, "has invalid characters"},
+	}
+
+	for _, tc := range testCases {
+		t.Run(tc.name, func(t *testing.T) {
+			err := Validate(tc.fieldName, tc.id)
+			if tc.wantErr {
+				assert.Error(t, err)
+				assert.Contains(t, err.Error(), tc.errContains)
+			} else {
+				assert.NoError(t, err)
+			}
+		})
+	}
+}
+
+func TestValidateWithPrefix(t *testing.T) {
+	prefix := "prefix"
+	validIdWithPrefix, _ := NewWithPrefix(prefix)
+
+	testCases := []struct {
+		name        string
+		fieldName   string
+		id          string
+		prefix      string
+		wantErr     bool
+		errContains string
+	}{
+		{"Valid ID with Prefix", "field", validIdWithPrefix, prefix, false, ""},
+		{"Empty ID", "field", "", prefix, true, "cannot be blank"},
+		{"No Prefix", "field", validIdWithPrefix[len(prefix)+1:], prefix, true, "must start with the prefix"},
+		{"Wrong Prefix", "field", "wrong_" + validIdWithPrefix, prefix, true, "must start with the prefix"},
+		{"Invalid Characters with Prefix", "field", prefix + "_invalidchar!", prefix, true, "has invalid characters"},
+	}
+
+	for _, tc := range testCases {
+		t.Run(tc.name, func(t *testing.T) {
+			err := ValidateWithPrefix(tc.fieldName, tc.id, tc.prefix)
+			if tc.wantErr {
+				assert.Error(t, err)
+				assert.Contains(t, err.Error(), tc.errContains)
+			} else {
+				assert.NoError(t, err)
+			}
+		})
+	}
+}
diff --git a/pkg/targets/clickhouse/creator.go b/pkg/targets/clickhouse/creator.go
@@ -143,7 +143,9 @@ func createMetricsTable(conf *ClickhouseConfig, db *sqlx.DB, tableName string, f
 				tags_id         UInt32,
 				%s,
 				additional_tags String   DEFAULT ''
-			) ENGINE = MergeTree(created_date, (tags_id, created_at), 8192)
+			) ENGINE = MergeTree() PARTITION BY toYYYYMM(created_date)
+			ORDER BY (tags_id, created_at)
+			SETTINGS index_granularity = 8192;
 			`,
 		tableName,
 		strings.Join(columnsWithType, ","))
@@ -180,7 +182,9 @@ func generateTagsTableQuery(tagNames, tagTypes []string) string {
 			"created_at   DateTime DEFAULT now(),\n"+
 			"id           UInt32,\n"+
 			"%s"+
-			") ENGINE = MergeTree(created_date, (%s), 8192)",
+			") ENGINE = MergeTree() PARTITION BY toYYYYMM(created_date)"+
+			"ORDER BY (%s)"+
+			"SETTINGS index_granularity = 8192;",
 		cols,
 		index)
 }