diff --git a/authenticator/env_gen.json b/authenticator/env_gen.json new file mode 100644 index 000000000..b36d7b62e --- /dev/null +++ b/authenticator/env_gen.json @@ -0,0 +1 @@ +[{"Category":"DEVTRON","Fields":[{"Env":"DEVTRON_DEFAULT_NAMESPACE","EnvType":"string","EnvValue":"devtroncd","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"DEVTRON_SECRET_NAME","EnvType":"string","EnvValue":"devtron-secret","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"DEX_CLIENT_ID","EnvType":"string","EnvValue":"argo-cd","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"DEX_HOST","EnvType":"string","EnvValue":"http://localhost","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"DEX_PORT","EnvType":"string","EnvValue":"5556","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"RUNTIME_CONFIG_LOCAL_DEV","EnvType":"LocalDevMode","EnvValue":"false","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"USER_SESSION_DURATION_SECONDS","EnvType":"int","EnvValue":"86400","EnvDescription":"","Example":"","Deprecated":"false"}]}] \ No newline at end of file diff --git a/authenticator/env_gen.md b/authenticator/env_gen.md new file mode 100644 index 000000000..f0f6a08d9 --- /dev/null +++ b/authenticator/env_gen.md @@ -0,0 +1,13 @@ + + +## DEVTRON Related Environment Variables +| Key | Type | Default Value | Description | Example | Deprecated | +|-------|----------|-------------------|-------------------|-----------------------|------------------| + | DEVTRON_DEFAULT_NAMESPACE | string |devtroncd | | | false | + | DEVTRON_SECRET_NAME | string |devtron-secret | | | false | + | DEX_CLIENT_ID | string |argo-cd | | | false | + | DEX_HOST | string |http://localhost | | | false | + | DEX_PORT | string |5556 | | | false | + | RUNTIME_CONFIG_LOCAL_DEV | LocalDevMode |false | | | false | + | USER_SESSION_DURATION_SECONDS | int |86400 | | | false | + diff --git a/chart-sync/Makefile b/chart-sync/Makefile index 3e0c99669..c6948e4e6 100644 --- a/chart-sync/Makefile +++ b/chart-sync/Makefile @@ -1,5 +1,5 @@ -all: build +all: fetch-all-env build TAG?=$(shell bash -c 'git log --pretty=format:'%h' -n 1') FLAGS= @@ -40,3 +40,6 @@ dep-update-oss: go mod edit -replace=github.com/devtron-labs/common-lib=github.com/devtron-labs/devtron-services/common-lib@$(TARGET_BRANCH) go mod tidy go mod vendor + +fetch-all-env: + go run fetchAllEnv/fetchAllEnv.go diff --git a/chart-sync/env_gen.json b/chart-sync/env_gen.json new file mode 100644 index 000000000..96f743d6a --- /dev/null +++ b/chart-sync/env_gen.json @@ -0,0 +1 @@ +[{"Category":"DEVTRON","Fields":[{"Env":"APP","EnvType":"string","EnvValue":"chart-sync","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"APP_STORE_APPLICATION_VERSIONS_SAVE_CHUNK_SIZE","EnvType":"int","EnvValue":"20","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"CHART_PROVIDER_ID","EnvType":"string","EnvValue":"*","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"IS_OCI_REGISTRY","EnvType":"bool","EnvValue":"true","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"LOG_LEVEL","EnvType":"int","EnvValue":"0","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"PARALLELISM_LIMIT_FOR_TAG_PROCESSING","EnvType":"int","EnvValue":"0","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"PG_ADDR","EnvType":"string","EnvValue":"127.0.0.1","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"PG_DATABASE","EnvType":"string","EnvValue":"orchestrator","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"PG_EXPORT_PROM_METRICS","EnvType":"bool","EnvValue":"true","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"PG_LOG_ALL_FAILURE_QUERIES","EnvType":"bool","EnvValue":"true","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"PG_LOG_ALL_QUERY","EnvType":"bool","EnvValue":"false","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"PG_LOG_SLOW_QUERY","EnvType":"bool","EnvValue":"true","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"PG_PASSWORD","EnvType":"string","EnvValue":"password","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"PG_PORT","EnvType":"string","EnvValue":"5432","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"PG_QUERY_DUR_THRESHOLD","EnvType":"int64","EnvValue":"5000","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"PG_USER","EnvType":"string","EnvValue":"user","EnvDescription":"","Example":"","Deprecated":"false"}]}] \ No newline at end of file diff --git a/chart-sync/env_gen.md b/chart-sync/env_gen.md new file mode 100644 index 000000000..4d1fde9e3 --- /dev/null +++ b/chart-sync/env_gen.md @@ -0,0 +1,22 @@ + + +## DEVTRON Related Environment Variables +| Key | Type | Default Value | Description | Example | Deprecated | +|-------|----------|-------------------|-------------------|-----------------------|------------------| + | APP | string |chart-sync | | | false | + | APP_STORE_APPLICATION_VERSIONS_SAVE_CHUNK_SIZE | int |20 | | | false | + | CHART_PROVIDER_ID | string |* | | | false | + | IS_OCI_REGISTRY | bool |true | | | false | + | LOG_LEVEL | int |0 | | | false | + | PARALLELISM_LIMIT_FOR_TAG_PROCESSING | int |0 | | | false | + | PG_ADDR | string |127.0.0.1 | | | false | + | PG_DATABASE | string |orchestrator | | | false | + | PG_EXPORT_PROM_METRICS | bool |true | | | false | + | PG_LOG_ALL_FAILURE_QUERIES | bool |true | | | false | + | PG_LOG_ALL_QUERY | bool |false | | | false | + | PG_LOG_SLOW_QUERY | bool |true | | | false | + | PG_PASSWORD | string |password | | | false | + | PG_PORT | string |5432 | | | false | + | PG_QUERY_DUR_THRESHOLD | int64 |5000 | | | false | + | PG_USER | string |user | | | false | + diff --git a/chart-sync/fetchAllEnv/fetchAllEnv.go b/chart-sync/fetchAllEnv/fetchAllEnv.go new file mode 100644 index 000000000..ed6163dc6 --- /dev/null +++ b/chart-sync/fetchAllEnv/fetchAllEnv.go @@ -0,0 +1,25 @@ +/* + * Copyright (c) 2024. Devtron Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package main + +import ( + "github.com/devtron-labs/common-lib/fetchAllEnv" +) + +func main() { + fetchAllEnv.FetchEnvAndWriteToFile() +} diff --git a/chart-sync/go.mod b/chart-sync/go.mod index 546c03ea5..fdcd19d43 100644 --- a/chart-sync/go.mod +++ b/chart-sync/go.mod @@ -5,7 +5,7 @@ go 1.22.4 toolchain go1.22.6 replace ( - github.com/devtron-labs/common-lib => github.com/devtron-labs/devtron-services/common-lib v0.0.0-20250127074233-945aa5b576dd + github.com/devtron-labs/common-lib => github.com/devtron-labs/devtron-services/common-lib v0.0.0-20250127103321-97c55f5baa57 helm.sh/helm/v3 v3.14.3 => github.com/devtron-labs/helm/v3 v3.14.1-0.20240401080259-90238cf69e42 ) diff --git a/chart-sync/go.sum b/chart-sync/go.sum index 17edd94f5..fb329b6b4 100644 --- a/chart-sync/go.sum +++ b/chart-sync/go.sum @@ -54,8 +54,8 @@ github.com/creack/pty v1.1.18/go.mod h1:MOBLtS5ELjhRRrroQr9kyvTxUAFNvYEK993ew/Vr github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= -github.com/devtron-labs/devtron-services/common-lib v0.0.0-20250127074233-945aa5b576dd h1:Mq9sBScfBWU4FWnfihutOXQbvy3t63tViYN9DVrPExw= -github.com/devtron-labs/devtron-services/common-lib v0.0.0-20250127074233-945aa5b576dd/go.mod h1:1QJJLpgJSkb5Jm9xPeKAk+kXb0QgBOOOgJj0cgYhAVA= +github.com/devtron-labs/devtron-services/common-lib v0.0.0-20250127103321-97c55f5baa57 h1:EPMILz/Lk4oK/gTJWiOL5a7jFAiuiokeFFc/T7qeQ/o= +github.com/devtron-labs/devtron-services/common-lib v0.0.0-20250127103321-97c55f5baa57/go.mod h1:1QJJLpgJSkb5Jm9xPeKAk+kXb0QgBOOOgJj0cgYhAVA= github.com/devtron-labs/helm/v3 v3.14.1-0.20240401080259-90238cf69e42 h1:pJmK44QaSztOiZe0iQHNf0sdy5KwkAeceydyhOG4RaY= github.com/devtron-labs/helm/v3 v3.14.1-0.20240401080259-90238cf69e42/go.mod h1:v6myVbyseSBJTzhmeE39UcPLNv6cQK6qss3dvgAySaE= github.com/dgryski/go-rendezvous v0.0.0-20200823014737-9f7001d12a5f h1:lO4WD4F/rVNCu3HqELle0jiPLLBs70cWOduZpkS1E78= diff --git a/chart-sync/vendor/github.com/devtron-labs/common-lib/fetchAllEnv/fetchAllEnv.go b/chart-sync/vendor/github.com/devtron-labs/common-lib/fetchAllEnv/fetchAllEnv.go new file mode 100644 index 000000000..c8a19e282 --- /dev/null +++ b/chart-sync/vendor/github.com/devtron-labs/common-lib/fetchAllEnv/fetchAllEnv.go @@ -0,0 +1,226 @@ +/* + * Copyright (c) 2024. Devtron Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package fetchAllEnv + +import ( + "encoding/json" + "errors" + "go/ast" + "go/parser" + "go/token" + "log" + "os" + "path/filepath" + "reflect" + "sort" + "strings" + "text/template" +) + +type EnvField struct { + Env string + EnvType string + EnvValue string + EnvDescription string + Example string + Deprecated string +} + +type CategoryField struct { + Category string + Fields []EnvField +} + +const ( + categoryCommentStructPrefix = "CATEGORY=" + defaultCategory = "DEVTRON" + deprecatedDefaultValue = "false" + + envFieldTypeTag = "env" + envDefaultFieldTypeTag = "envDefault" + envDescriptionFieldTypeTag = "description" + envPossibleValuesFieldTypeTag = "example" + envDeprecatedFieldTypeTag = "deprecated" + MARKDOWN_FILENAME = "env_gen.md" + MARKDOWN_JSON_FILENAME = "env_gen.json" +) + +const MarkdownTemplate = ` +{{range . }} +## {{ .Category }} Related Environment Variables +| Key | Type | Default Value | Description | Example | Deprecated | +|-------|----------|-------------------|-------------------|-----------------------|------------------| +{{range .Fields }} | {{ .Env }} | {{ .EnvType }} |{{ .EnvValue }} | {{ .EnvDescription }} | {{ .Example }} | {{ .Deprecated }} | +{{end}} +{{end}}` + +func FetchEnvAndWriteToFile() { + WalkThroughProject() + return +} + +func WalkThroughProject() { + categoryFieldsMap := make(map[string][]EnvField) + uniqueKeys := make(map[string]bool) + err := filepath.Walk(".", func(path string, info os.FileInfo, err error) error { + if err != nil { + return err + } + if !info.IsDir() && strings.HasSuffix(path, ".go") { + err = processGoFile(path, categoryFieldsMap, uniqueKeys) + if err != nil { + log.Println("error in processing go file", err) + return err + } + } + return nil + }) + if err != nil { + return + } + writeToFile(categoryFieldsMap) +} + +func processGoFile(filePath string, categoryFieldsMap map[string][]EnvField, uniqueKeys map[string]bool) error { + fset := token.NewFileSet() + node, err := parser.ParseFile(fset, filePath, nil, parser.ParseComments) + if err != nil { + log.Println("error parsing file:", err) + return err + } + ast.Inspect(node, func(n ast.Node) bool { + if genDecl, ok := n.(*ast.GenDecl); ok { + // checking if type declaration, one of [func, map, struct, array, channel, interface] + if genDecl.Tok == token.TYPE { + for _, spec := range genDecl.Specs { + if typeSpec, ok := spec.(*ast.TypeSpec); ok { + // only checking struct type declarations + if structType, ok2 := typeSpec.Type.(*ast.StructType); ok2 { + allFields := make([]EnvField, 0, len(structType.Fields.List)) + for _, field := range structType.Fields.List { + if field.Tag != nil { + envField := getEnvKeyAndValue(field) + envKey := envField.Env + if len(envKey) == 0 || uniqueKeys[envKey] { + continue + } + allFields = append(allFields, envField) + uniqueKeys[envKey] = true + } + } + if len(allFields) > 0 { + category := getCategoryForAStruct(genDecl) + categoryFieldsMap[category] = append(categoryFieldsMap[category], allFields...) + } + } + } + } + } + } + return true + }) + return nil +} + +func getEnvKeyAndValue(field *ast.Field) EnvField { + tag := reflect.StructTag(strings.Trim(field.Tag.Value, "`")) // remove surrounding backticks + + envKey := addReadmeTableDelimiterEscapeChar(tag.Get(envFieldTypeTag)) + envValue := addReadmeTableDelimiterEscapeChar(tag.Get(envDefaultFieldTypeTag)) + envDescription := addReadmeTableDelimiterEscapeChar(tag.Get(envDescriptionFieldTypeTag)) + envPossibleValues := addReadmeTableDelimiterEscapeChar(tag.Get(envPossibleValuesFieldTypeTag)) + envDeprecated := addReadmeTableDelimiterEscapeChar(tag.Get(envDeprecatedFieldTypeTag)) + // check if there exist any value provided in env for this field + if value, ok := os.LookupEnv(envKey); ok { + envValue = value + } + env := EnvField{ + Env: envKey, + EnvValue: envValue, + EnvDescription: envDescription, + Example: envPossibleValues, + Deprecated: envDeprecated, + } + if indent, ok := field.Type.(*ast.Ident); ok && indent != nil { + env.EnvType = indent.Name + } + if len(envDeprecated) == 0 { + env.Deprecated = deprecatedDefaultValue + } + return env +} + +func getCategoryForAStruct(genDecl *ast.GenDecl) string { + category := defaultCategory + if genDecl.Doc != nil { + commentTexts := strings.Split(genDecl.Doc.Text(), "\n") + for _, comment := range commentTexts { + commentText := strings.TrimPrefix(strings.ReplaceAll(comment, " ", ""), "//") // this can happen if comment group is in /* */ + if strings.HasPrefix(commentText, categoryCommentStructPrefix) { + categories := strings.Split(strings.TrimPrefix(commentText, categoryCommentStructPrefix), ",") + if len(categories) > 0 && len(categories[0]) > 0 { //only supporting one category as of now + category = categories[0] //overriding category + break + } + } + } + } + return category +} + +func addReadmeTableDelimiterEscapeChar(s string) string { + return strings.ReplaceAll(s, "|", `\|`) +} + +func writeToFile(categoryFieldsMap map[string][]EnvField) { + cfs := make([]CategoryField, 0, len(categoryFieldsMap)) + for category, allFields := range categoryFieldsMap { + sort.Slice(allFields, func(i, j int) bool { + return allFields[i].Env < allFields[j].Env + }) + + cfs = append(cfs, CategoryField{ + Category: category, + Fields: allFields, + }) + } + sort.Slice(cfs, func(i, j int) bool { + return cfs[i].Category < cfs[j].Category + }) + file, err := os.Create(MARKDOWN_FILENAME) + if err != nil && !errors.Is(err, os.ErrExist) { + panic(err) + } + defer file.Close() + tmpl, err := template.New("markdown").Parse(MarkdownTemplate) + if err != nil { + panic(err) + } + err = tmpl.Execute(file, cfs) + if err != nil { + panic(err) + } + cfsMarshaled, err := json.Marshal(cfs) + if err != nil { + log.Println("error marshalling category fields:", err) + panic(err) + } + err = os.WriteFile(MARKDOWN_JSON_FILENAME, cfsMarshaled, 0644) + if err != nil { + panic(err) + } +} diff --git a/chart-sync/vendor/modules.txt b/chart-sync/vendor/modules.txt index 5ba93f7c3..5e3b481c9 100644 --- a/chart-sync/vendor/modules.txt +++ b/chart-sync/vendor/modules.txt @@ -93,8 +93,9 @@ github.com/containerd/platforms # github.com/davecgh/go-spew v1.1.1 ## explicit github.com/davecgh/go-spew/spew -# github.com/devtron-labs/common-lib v0.0.0 => github.com/devtron-labs/devtron-services/common-lib v0.0.0-20250127074233-945aa5b576dd +# github.com/devtron-labs/common-lib v0.0.0 => github.com/devtron-labs/devtron-services/common-lib v0.0.0-20250127103321-97c55f5baa57 ## explicit; go 1.21 +github.com/devtron-labs/common-lib/fetchAllEnv github.com/devtron-labs/common-lib/git-manager/util github.com/devtron-labs/common-lib/helmLib/registry github.com/devtron-labs/common-lib/utils @@ -789,4 +790,4 @@ sigs.k8s.io/structured-merge-diff/v4/value # sigs.k8s.io/yaml v1.3.0 ## explicit; go 1.12 sigs.k8s.io/yaml -# github.com/devtron-labs/common-lib => github.com/devtron-labs/devtron-services/common-lib v0.0.0-20250127074233-945aa5b576dd +# github.com/devtron-labs/common-lib => github.com/devtron-labs/devtron-services/common-lib v0.0.0-20250127103321-97c55f5baa57 diff --git a/chart-sync/wire_gen.go b/chart-sync/wire_gen.go index 92e4203e8..a17c7ac6a 100644 --- a/chart-sync/wire_gen.go +++ b/chart-sync/wire_gen.go @@ -1,6 +1,6 @@ // Code generated by Wire. DO NOT EDIT. -//go:generate go run github.com/google/wire/cmd/wire +//go:generate go run -mod=mod github.com/google/wire/cmd/wire //go:build !wireinject // +build !wireinject diff --git a/ci-runner/Makefile b/ci-runner/Makefile index cd8071d1e..be72373d5 100644 --- a/ci-runner/Makefile +++ b/ci-runner/Makefile @@ -1,5 +1,5 @@ -all: build +all: fetch-all-env build TAG?=6bd516eacd6ee-10 FLAGS= @@ -34,4 +34,7 @@ docker-build-push: docker-build-image dep-update-oss: go mod edit -replace=github.com/devtron-labs/common-lib=github.com/devtron-labs/devtron-services/common-lib@$(TARGET_BRANCH) go mod tidy - go mod vendor \ No newline at end of file + go mod vendor + +fetch-all-env: + go run fetchAllEnv/fetchAllEnv.go diff --git a/ci-runner/env_gen.json b/ci-runner/env_gen.json new file mode 100644 index 000000000..5334f14fd --- /dev/null +++ b/ci-runner/env_gen.json @@ -0,0 +1 @@ +[{"Category":"DEVTRON","Fields":[{"Env":"AZURE_ACCOUNT_KEY","EnvType":"string","EnvValue":"","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"AZURE_ACCOUNT_NAME","EnvType":"string","EnvValue":"","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"AZURE_BLOB_CONTAINER_CI_CACHE","EnvType":"string","EnvValue":"","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"AZURE_BLOB_CONTAINER_CI_LOG","EnvType":"string","EnvValue":"","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"AZURE_GATEWAY_CONNECTION_INSECURE","EnvType":"bool","EnvValue":"true","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"AZURE_GATEWAY_URL","EnvType":"string","EnvValue":"","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"BLOB_STORAGE_GCP_CREDENTIALS_JSON","EnvType":"string","EnvValue":"","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"BLOB_STORAGE_PROVIDER","EnvType":"","EnvValue":"","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"BLOB_STORAGE_S3_ACCESS_KEY","EnvType":"string","EnvValue":"","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"BLOB_STORAGE_S3_BUCKET_VERSIONED","EnvType":"bool","EnvValue":"true","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"BLOB_STORAGE_S3_ENDPOINT","EnvType":"string","EnvValue":"","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"BLOB_STORAGE_S3_ENDPOINT_INSECURE","EnvType":"bool","EnvValue":"false","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"BLOB_STORAGE_S3_SECRET_KEY","EnvType":"string","EnvValue":"","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"CONSUMER_CONFIG_JSON","EnvType":"string","EnvValue":"","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"DEFAULT_BUILD_LOGS_BUCKET","EnvType":"string","EnvValue":"","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"DEFAULT_CACHE_BUCKET","EnvType":"string","EnvValue":"","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"DEFAULT_CACHE_BUCKET_REGION","EnvType":"string","EnvValue":"","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"DEFAULT_CD_LOGS_BUCKET_REGION","EnvType":"string","EnvValue":"","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"DEFAULT_LOG_TIME_LIMIT","EnvType":"int64","EnvValue":"1","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"IMAGE_SCANNER_ENDPOINT","EnvType":"string","EnvValue":"http://image-scanner-new-demo-devtroncd-service.devtroncd:80","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"LOG_LEVEL","EnvType":"int","EnvValue":"0","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"NATS_MSG_ACK_WAIT_IN_SECS","EnvType":"int","EnvValue":"120","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"NATS_MSG_BUFFER_SIZE","EnvType":"int","EnvValue":"-1","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"NATS_MSG_MAX_AGE","EnvType":"int","EnvValue":"86400","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"NATS_MSG_PROCESSING_BATCH_SIZE","EnvType":"int","EnvValue":"1","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"NATS_MSG_REPLICAS","EnvType":"int","EnvValue":"0","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"NATS_SERVER_HOST","EnvType":"string","EnvValue":"nats://devtron-nats.devtroncd:4222","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"PG_EXPORT_PROM_METRICS","EnvType":"bool","EnvValue":"true","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"PG_LOG_ALL_FAILURE_QUERIES","EnvType":"bool","EnvValue":"true","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"PG_LOG_ALL_QUERY","EnvType":"bool","EnvValue":"false","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"PG_LOG_SLOW_QUERY","EnvType":"bool","EnvValue":"true","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"PG_QUERY_DUR_THRESHOLD","EnvType":"int64","EnvValue":"5000","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"SHOW_DOCKER_BUILD_ARGS","EnvType":"bool","EnvValue":"true","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"STREAM_CONFIG_JSON","EnvType":"string","EnvValue":"","EnvDescription":"","Example":"","Deprecated":"false"}]}] \ No newline at end of file diff --git a/ci-runner/env_gen.md b/ci-runner/env_gen.md new file mode 100644 index 000000000..033d7a119 --- /dev/null +++ b/ci-runner/env_gen.md @@ -0,0 +1,40 @@ + + +## DEVTRON Related Environment Variables +| Key | Type | Default Value | Description | Example | Deprecated | +|-------|----------|-------------------|-------------------|-----------------------|------------------| + | AZURE_ACCOUNT_KEY | string | | | | false | + | AZURE_ACCOUNT_NAME | string | | | | false | + | AZURE_BLOB_CONTAINER_CI_CACHE | string | | | | false | + | AZURE_BLOB_CONTAINER_CI_LOG | string | | | | false | + | AZURE_GATEWAY_CONNECTION_INSECURE | bool |true | | | false | + | AZURE_GATEWAY_URL | string | | | | false | + | BLOB_STORAGE_GCP_CREDENTIALS_JSON | string | | | | false | + | BLOB_STORAGE_PROVIDER | | | | | false | + | BLOB_STORAGE_S3_ACCESS_KEY | string | | | | false | + | BLOB_STORAGE_S3_BUCKET_VERSIONED | bool |true | | | false | + | BLOB_STORAGE_S3_ENDPOINT | string | | | | false | + | BLOB_STORAGE_S3_ENDPOINT_INSECURE | bool |false | | | false | + | BLOB_STORAGE_S3_SECRET_KEY | string | | | | false | + | CONSUMER_CONFIG_JSON | string | | | | false | + | DEFAULT_BUILD_LOGS_BUCKET | string | | | | false | + | DEFAULT_CACHE_BUCKET | string | | | | false | + | DEFAULT_CACHE_BUCKET_REGION | string | | | | false | + | DEFAULT_CD_LOGS_BUCKET_REGION | string | | | | false | + | DEFAULT_LOG_TIME_LIMIT | int64 |1 | | | false | + | IMAGE_SCANNER_ENDPOINT | string |http://image-scanner-new-demo-devtroncd-service.devtroncd:80 | | | false | + | LOG_LEVEL | int |0 | | | false | + | NATS_MSG_ACK_WAIT_IN_SECS | int |120 | | | false | + | NATS_MSG_BUFFER_SIZE | int |-1 | | | false | + | NATS_MSG_MAX_AGE | int |86400 | | | false | + | NATS_MSG_PROCESSING_BATCH_SIZE | int |1 | | | false | + | NATS_MSG_REPLICAS | int |0 | | | false | + | NATS_SERVER_HOST | string |nats://devtron-nats.devtroncd:4222 | | | false | + | PG_EXPORT_PROM_METRICS | bool |true | | | false | + | PG_LOG_ALL_FAILURE_QUERIES | bool |true | | | false | + | PG_LOG_ALL_QUERY | bool |false | | | false | + | PG_LOG_SLOW_QUERY | bool |true | | | false | + | PG_QUERY_DUR_THRESHOLD | int64 |5000 | | | false | + | SHOW_DOCKER_BUILD_ARGS | bool |true | | | false | + | STREAM_CONFIG_JSON | string | | | | false | + diff --git a/ci-runner/fetchAllEnv/fetchAllEnv.go b/ci-runner/fetchAllEnv/fetchAllEnv.go new file mode 100644 index 000000000..ed6163dc6 --- /dev/null +++ b/ci-runner/fetchAllEnv/fetchAllEnv.go @@ -0,0 +1,25 @@ +/* + * Copyright (c) 2024. Devtron Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package main + +import ( + "github.com/devtron-labs/common-lib/fetchAllEnv" +) + +func main() { + fetchAllEnv.FetchEnvAndWriteToFile() +} diff --git a/ci-runner/go.mod b/ci-runner/go.mod index daa2a6d80..2084a34d0 100644 --- a/ci-runner/go.mod +++ b/ci-runner/go.mod @@ -4,7 +4,7 @@ go 1.21 toolchain go1.21.8 -replace github.com/devtron-labs/common-lib => github.com/devtron-labs/devtron-services/common-lib v0.0.0-20250127074233-945aa5b576dd +replace github.com/devtron-labs/common-lib => github.com/devtron-labs/devtron-services/common-lib v0.0.0-20250127103321-97c55f5baa57 require ( github.com/Knetic/govaluate v3.0.0+incompatible diff --git a/ci-runner/go.sum b/ci-runner/go.sum index b8f18ec3d..6f63edb1f 100644 --- a/ci-runner/go.sum +++ b/ci-runner/go.sum @@ -57,8 +57,8 @@ github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSs github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc h1:U9qPSI2PIWSS1VwoXQT9A3Wy9MM3WgvqSxFWenqJduM= github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= -github.com/devtron-labs/devtron-services/common-lib v0.0.0-20250127074233-945aa5b576dd h1:Mq9sBScfBWU4FWnfihutOXQbvy3t63tViYN9DVrPExw= -github.com/devtron-labs/devtron-services/common-lib v0.0.0-20250127074233-945aa5b576dd/go.mod h1:1QJJLpgJSkb5Jm9xPeKAk+kXb0QgBOOOgJj0cgYhAVA= +github.com/devtron-labs/devtron-services/common-lib v0.0.0-20250127103321-97c55f5baa57 h1:EPMILz/Lk4oK/gTJWiOL5a7jFAiuiokeFFc/T7qeQ/o= +github.com/devtron-labs/devtron-services/common-lib v0.0.0-20250127103321-97c55f5baa57/go.mod h1:1QJJLpgJSkb5Jm9xPeKAk+kXb0QgBOOOgJj0cgYhAVA= github.com/distribution/reference v0.6.0 h1:0IXCQ5g4/QMHHkarYzh5l+u8T3t73zM5QvfrDyIgxBk= github.com/distribution/reference v0.6.0/go.mod h1:BbU0aIcezP1/5jX/8MP0YiH4SdvB5Y4f/wlDRiLyi3E= github.com/docker/cli v24.0.6+incompatible h1:fF+XCQCgJjjQNIMjzaSmiKJSCcfcXb3TWTcc7GAneOY= diff --git a/ci-runner/vendor/github.com/devtron-labs/common-lib/fetchAllEnv/fetchAllEnv.go b/ci-runner/vendor/github.com/devtron-labs/common-lib/fetchAllEnv/fetchAllEnv.go new file mode 100644 index 000000000..c8a19e282 --- /dev/null +++ b/ci-runner/vendor/github.com/devtron-labs/common-lib/fetchAllEnv/fetchAllEnv.go @@ -0,0 +1,226 @@ +/* + * Copyright (c) 2024. Devtron Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package fetchAllEnv + +import ( + "encoding/json" + "errors" + "go/ast" + "go/parser" + "go/token" + "log" + "os" + "path/filepath" + "reflect" + "sort" + "strings" + "text/template" +) + +type EnvField struct { + Env string + EnvType string + EnvValue string + EnvDescription string + Example string + Deprecated string +} + +type CategoryField struct { + Category string + Fields []EnvField +} + +const ( + categoryCommentStructPrefix = "CATEGORY=" + defaultCategory = "DEVTRON" + deprecatedDefaultValue = "false" + + envFieldTypeTag = "env" + envDefaultFieldTypeTag = "envDefault" + envDescriptionFieldTypeTag = "description" + envPossibleValuesFieldTypeTag = "example" + envDeprecatedFieldTypeTag = "deprecated" + MARKDOWN_FILENAME = "env_gen.md" + MARKDOWN_JSON_FILENAME = "env_gen.json" +) + +const MarkdownTemplate = ` +{{range . }} +## {{ .Category }} Related Environment Variables +| Key | Type | Default Value | Description | Example | Deprecated | +|-------|----------|-------------------|-------------------|-----------------------|------------------| +{{range .Fields }} | {{ .Env }} | {{ .EnvType }} |{{ .EnvValue }} | {{ .EnvDescription }} | {{ .Example }} | {{ .Deprecated }} | +{{end}} +{{end}}` + +func FetchEnvAndWriteToFile() { + WalkThroughProject() + return +} + +func WalkThroughProject() { + categoryFieldsMap := make(map[string][]EnvField) + uniqueKeys := make(map[string]bool) + err := filepath.Walk(".", func(path string, info os.FileInfo, err error) error { + if err != nil { + return err + } + if !info.IsDir() && strings.HasSuffix(path, ".go") { + err = processGoFile(path, categoryFieldsMap, uniqueKeys) + if err != nil { + log.Println("error in processing go file", err) + return err + } + } + return nil + }) + if err != nil { + return + } + writeToFile(categoryFieldsMap) +} + +func processGoFile(filePath string, categoryFieldsMap map[string][]EnvField, uniqueKeys map[string]bool) error { + fset := token.NewFileSet() + node, err := parser.ParseFile(fset, filePath, nil, parser.ParseComments) + if err != nil { + log.Println("error parsing file:", err) + return err + } + ast.Inspect(node, func(n ast.Node) bool { + if genDecl, ok := n.(*ast.GenDecl); ok { + // checking if type declaration, one of [func, map, struct, array, channel, interface] + if genDecl.Tok == token.TYPE { + for _, spec := range genDecl.Specs { + if typeSpec, ok := spec.(*ast.TypeSpec); ok { + // only checking struct type declarations + if structType, ok2 := typeSpec.Type.(*ast.StructType); ok2 { + allFields := make([]EnvField, 0, len(structType.Fields.List)) + for _, field := range structType.Fields.List { + if field.Tag != nil { + envField := getEnvKeyAndValue(field) + envKey := envField.Env + if len(envKey) == 0 || uniqueKeys[envKey] { + continue + } + allFields = append(allFields, envField) + uniqueKeys[envKey] = true + } + } + if len(allFields) > 0 { + category := getCategoryForAStruct(genDecl) + categoryFieldsMap[category] = append(categoryFieldsMap[category], allFields...) + } + } + } + } + } + } + return true + }) + return nil +} + +func getEnvKeyAndValue(field *ast.Field) EnvField { + tag := reflect.StructTag(strings.Trim(field.Tag.Value, "`")) // remove surrounding backticks + + envKey := addReadmeTableDelimiterEscapeChar(tag.Get(envFieldTypeTag)) + envValue := addReadmeTableDelimiterEscapeChar(tag.Get(envDefaultFieldTypeTag)) + envDescription := addReadmeTableDelimiterEscapeChar(tag.Get(envDescriptionFieldTypeTag)) + envPossibleValues := addReadmeTableDelimiterEscapeChar(tag.Get(envPossibleValuesFieldTypeTag)) + envDeprecated := addReadmeTableDelimiterEscapeChar(tag.Get(envDeprecatedFieldTypeTag)) + // check if there exist any value provided in env for this field + if value, ok := os.LookupEnv(envKey); ok { + envValue = value + } + env := EnvField{ + Env: envKey, + EnvValue: envValue, + EnvDescription: envDescription, + Example: envPossibleValues, + Deprecated: envDeprecated, + } + if indent, ok := field.Type.(*ast.Ident); ok && indent != nil { + env.EnvType = indent.Name + } + if len(envDeprecated) == 0 { + env.Deprecated = deprecatedDefaultValue + } + return env +} + +func getCategoryForAStruct(genDecl *ast.GenDecl) string { + category := defaultCategory + if genDecl.Doc != nil { + commentTexts := strings.Split(genDecl.Doc.Text(), "\n") + for _, comment := range commentTexts { + commentText := strings.TrimPrefix(strings.ReplaceAll(comment, " ", ""), "//") // this can happen if comment group is in /* */ + if strings.HasPrefix(commentText, categoryCommentStructPrefix) { + categories := strings.Split(strings.TrimPrefix(commentText, categoryCommentStructPrefix), ",") + if len(categories) > 0 && len(categories[0]) > 0 { //only supporting one category as of now + category = categories[0] //overriding category + break + } + } + } + } + return category +} + +func addReadmeTableDelimiterEscapeChar(s string) string { + return strings.ReplaceAll(s, "|", `\|`) +} + +func writeToFile(categoryFieldsMap map[string][]EnvField) { + cfs := make([]CategoryField, 0, len(categoryFieldsMap)) + for category, allFields := range categoryFieldsMap { + sort.Slice(allFields, func(i, j int) bool { + return allFields[i].Env < allFields[j].Env + }) + + cfs = append(cfs, CategoryField{ + Category: category, + Fields: allFields, + }) + } + sort.Slice(cfs, func(i, j int) bool { + return cfs[i].Category < cfs[j].Category + }) + file, err := os.Create(MARKDOWN_FILENAME) + if err != nil && !errors.Is(err, os.ErrExist) { + panic(err) + } + defer file.Close() + tmpl, err := template.New("markdown").Parse(MarkdownTemplate) + if err != nil { + panic(err) + } + err = tmpl.Execute(file, cfs) + if err != nil { + panic(err) + } + cfsMarshaled, err := json.Marshal(cfs) + if err != nil { + log.Println("error marshalling category fields:", err) + panic(err) + } + err = os.WriteFile(MARKDOWN_JSON_FILENAME, cfsMarshaled, 0644) + if err != nil { + panic(err) + } +} diff --git a/ci-runner/vendor/modules.txt b/ci-runner/vendor/modules.txt index 27555820e..8cea9b005 100644 --- a/ci-runner/vendor/modules.txt +++ b/ci-runner/vendor/modules.txt @@ -118,10 +118,11 @@ github.com/cespare/xxhash/v2 # github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc ## explicit github.com/davecgh/go-spew/spew -# github.com/devtron-labs/common-lib v0.19.0 => github.com/devtron-labs/devtron-services/common-lib v0.0.0-20250127074233-945aa5b576dd +# github.com/devtron-labs/common-lib v0.19.0 => github.com/devtron-labs/devtron-services/common-lib v0.0.0-20250127103321-97c55f5baa57 ## explicit; go 1.21 github.com/devtron-labs/common-lib/blob-storage github.com/devtron-labs/common-lib/constants +github.com/devtron-labs/common-lib/fetchAllEnv github.com/devtron-labs/common-lib/git-manager github.com/devtron-labs/common-lib/git-manager/util github.com/devtron-labs/common-lib/imageScan/bean @@ -860,4 +861,4 @@ sigs.k8s.io/structured-merge-diff/v4/value # sigs.k8s.io/yaml v1.3.0 ## explicit; go 1.12 sigs.k8s.io/yaml -# github.com/devtron-labs/common-lib => github.com/devtron-labs/devtron-services/common-lib v0.0.0-20250127074233-945aa5b576dd +# github.com/devtron-labs/common-lib => github.com/devtron-labs/devtron-services/common-lib v0.0.0-20250127103321-97c55f5baa57 diff --git a/common-lib/fetchAllEnv/fetchAllEnv.go b/common-lib/fetchAllEnv/fetchAllEnv.go new file mode 100644 index 000000000..c8a19e282 --- /dev/null +++ b/common-lib/fetchAllEnv/fetchAllEnv.go @@ -0,0 +1,226 @@ +/* + * Copyright (c) 2024. Devtron Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package fetchAllEnv + +import ( + "encoding/json" + "errors" + "go/ast" + "go/parser" + "go/token" + "log" + "os" + "path/filepath" + "reflect" + "sort" + "strings" + "text/template" +) + +type EnvField struct { + Env string + EnvType string + EnvValue string + EnvDescription string + Example string + Deprecated string +} + +type CategoryField struct { + Category string + Fields []EnvField +} + +const ( + categoryCommentStructPrefix = "CATEGORY=" + defaultCategory = "DEVTRON" + deprecatedDefaultValue = "false" + + envFieldTypeTag = "env" + envDefaultFieldTypeTag = "envDefault" + envDescriptionFieldTypeTag = "description" + envPossibleValuesFieldTypeTag = "example" + envDeprecatedFieldTypeTag = "deprecated" + MARKDOWN_FILENAME = "env_gen.md" + MARKDOWN_JSON_FILENAME = "env_gen.json" +) + +const MarkdownTemplate = ` +{{range . }} +## {{ .Category }} Related Environment Variables +| Key | Type | Default Value | Description | Example | Deprecated | +|-------|----------|-------------------|-------------------|-----------------------|------------------| +{{range .Fields }} | {{ .Env }} | {{ .EnvType }} |{{ .EnvValue }} | {{ .EnvDescription }} | {{ .Example }} | {{ .Deprecated }} | +{{end}} +{{end}}` + +func FetchEnvAndWriteToFile() { + WalkThroughProject() + return +} + +func WalkThroughProject() { + categoryFieldsMap := make(map[string][]EnvField) + uniqueKeys := make(map[string]bool) + err := filepath.Walk(".", func(path string, info os.FileInfo, err error) error { + if err != nil { + return err + } + if !info.IsDir() && strings.HasSuffix(path, ".go") { + err = processGoFile(path, categoryFieldsMap, uniqueKeys) + if err != nil { + log.Println("error in processing go file", err) + return err + } + } + return nil + }) + if err != nil { + return + } + writeToFile(categoryFieldsMap) +} + +func processGoFile(filePath string, categoryFieldsMap map[string][]EnvField, uniqueKeys map[string]bool) error { + fset := token.NewFileSet() + node, err := parser.ParseFile(fset, filePath, nil, parser.ParseComments) + if err != nil { + log.Println("error parsing file:", err) + return err + } + ast.Inspect(node, func(n ast.Node) bool { + if genDecl, ok := n.(*ast.GenDecl); ok { + // checking if type declaration, one of [func, map, struct, array, channel, interface] + if genDecl.Tok == token.TYPE { + for _, spec := range genDecl.Specs { + if typeSpec, ok := spec.(*ast.TypeSpec); ok { + // only checking struct type declarations + if structType, ok2 := typeSpec.Type.(*ast.StructType); ok2 { + allFields := make([]EnvField, 0, len(structType.Fields.List)) + for _, field := range structType.Fields.List { + if field.Tag != nil { + envField := getEnvKeyAndValue(field) + envKey := envField.Env + if len(envKey) == 0 || uniqueKeys[envKey] { + continue + } + allFields = append(allFields, envField) + uniqueKeys[envKey] = true + } + } + if len(allFields) > 0 { + category := getCategoryForAStruct(genDecl) + categoryFieldsMap[category] = append(categoryFieldsMap[category], allFields...) + } + } + } + } + } + } + return true + }) + return nil +} + +func getEnvKeyAndValue(field *ast.Field) EnvField { + tag := reflect.StructTag(strings.Trim(field.Tag.Value, "`")) // remove surrounding backticks + + envKey := addReadmeTableDelimiterEscapeChar(tag.Get(envFieldTypeTag)) + envValue := addReadmeTableDelimiterEscapeChar(tag.Get(envDefaultFieldTypeTag)) + envDescription := addReadmeTableDelimiterEscapeChar(tag.Get(envDescriptionFieldTypeTag)) + envPossibleValues := addReadmeTableDelimiterEscapeChar(tag.Get(envPossibleValuesFieldTypeTag)) + envDeprecated := addReadmeTableDelimiterEscapeChar(tag.Get(envDeprecatedFieldTypeTag)) + // check if there exist any value provided in env for this field + if value, ok := os.LookupEnv(envKey); ok { + envValue = value + } + env := EnvField{ + Env: envKey, + EnvValue: envValue, + EnvDescription: envDescription, + Example: envPossibleValues, + Deprecated: envDeprecated, + } + if indent, ok := field.Type.(*ast.Ident); ok && indent != nil { + env.EnvType = indent.Name + } + if len(envDeprecated) == 0 { + env.Deprecated = deprecatedDefaultValue + } + return env +} + +func getCategoryForAStruct(genDecl *ast.GenDecl) string { + category := defaultCategory + if genDecl.Doc != nil { + commentTexts := strings.Split(genDecl.Doc.Text(), "\n") + for _, comment := range commentTexts { + commentText := strings.TrimPrefix(strings.ReplaceAll(comment, " ", ""), "//") // this can happen if comment group is in /* */ + if strings.HasPrefix(commentText, categoryCommentStructPrefix) { + categories := strings.Split(strings.TrimPrefix(commentText, categoryCommentStructPrefix), ",") + if len(categories) > 0 && len(categories[0]) > 0 { //only supporting one category as of now + category = categories[0] //overriding category + break + } + } + } + } + return category +} + +func addReadmeTableDelimiterEscapeChar(s string) string { + return strings.ReplaceAll(s, "|", `\|`) +} + +func writeToFile(categoryFieldsMap map[string][]EnvField) { + cfs := make([]CategoryField, 0, len(categoryFieldsMap)) + for category, allFields := range categoryFieldsMap { + sort.Slice(allFields, func(i, j int) bool { + return allFields[i].Env < allFields[j].Env + }) + + cfs = append(cfs, CategoryField{ + Category: category, + Fields: allFields, + }) + } + sort.Slice(cfs, func(i, j int) bool { + return cfs[i].Category < cfs[j].Category + }) + file, err := os.Create(MARKDOWN_FILENAME) + if err != nil && !errors.Is(err, os.ErrExist) { + panic(err) + } + defer file.Close() + tmpl, err := template.New("markdown").Parse(MarkdownTemplate) + if err != nil { + panic(err) + } + err = tmpl.Execute(file, cfs) + if err != nil { + panic(err) + } + cfsMarshaled, err := json.Marshal(cfs) + if err != nil { + log.Println("error marshalling category fields:", err) + panic(err) + } + err = os.WriteFile(MARKDOWN_JSON_FILENAME, cfsMarshaled, 0644) + if err != nil { + panic(err) + } +} diff --git a/git-sensor/Makefile b/git-sensor/Makefile index f3f8589ce..d409dc18c 100644 --- a/git-sensor/Makefile +++ b/git-sensor/Makefile @@ -1,5 +1,5 @@ -all: build +all: fetch-all-env build TAG?=latest FLAGS= @@ -43,3 +43,7 @@ dep-update-oss: go mod vendor +fetch-all-env: + go run fetchAllEnv/fetchAllEnv.go + + diff --git a/git-sensor/env_gen.json b/git-sensor/env_gen.json new file mode 100644 index 000000000..2a7cbdb4f --- /dev/null +++ b/git-sensor/env_gen.json @@ -0,0 +1 @@ +[{"Category":"DEVTRON","Fields":[{"Env":"ANALYTICS_DEBUG","EnvType":"bool","EnvValue":"false","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"APP","EnvType":"string","EnvValue":"git-sensor","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"CLI_CMD_TIMEOUT_GLOBAL_SECONDS","EnvType":"int","EnvValue":"900","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"CLI_CMD_TIMEOUT_JSON","EnvType":"string","EnvValue":"","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"COMMIT_STATS_TIMEOUT_IN_SEC","EnvType":"int","EnvValue":"2","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"CONSUMER_CONFIG_JSON","EnvType":"string","EnvValue":"","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"DEFAULT_LOG_TIME_LIMIT","EnvType":"int64","EnvValue":"1","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"ENABLE_FILE_STATS","EnvType":"bool","EnvValue":"false","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"ENABLE_STATSVIZ","EnvType":"bool","EnvValue":"false","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"GIT_HISTORY_COUNT","EnvType":"int","EnvValue":"15","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"GOGIT_TIMEOUT_SECONDS","EnvType":"int","EnvValue":"10","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"LOG_LEVEL","EnvType":"int","EnvValue":"0","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"MIN_LIMIT_FOR_PVC","EnvType":"int","EnvValue":"1","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"NATS_MSG_ACK_WAIT_IN_SECS","EnvType":"int","EnvValue":"120","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"NATS_MSG_BUFFER_SIZE","EnvType":"int","EnvValue":"-1","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"NATS_MSG_MAX_AGE","EnvType":"int","EnvValue":"86400","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"NATS_MSG_PROCESSING_BATCH_SIZE","EnvType":"int","EnvValue":"1","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"NATS_MSG_REPLICAS","EnvType":"int","EnvValue":"0","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"NATS_SERVER_HOST","EnvType":"string","EnvValue":"nats://devtron-nats.devtroncd:4222","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"PG_ADDR","EnvType":"string","EnvValue":"127.0.0.1","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"PG_DATABASE","EnvType":"string","EnvValue":"git_sensor","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"PG_EXPORT_PROM_METRICS","EnvType":"bool","EnvValue":"true","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"PG_LOG_ALL_FAILURE_QUERIES","EnvType":"bool","EnvValue":"true","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"PG_LOG_ALL_QUERY","EnvType":"bool","EnvValue":"false","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"PG_LOG_SLOW_QUERY","EnvType":"bool","EnvValue":"true","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"PG_PASSWORD","EnvType":"string","EnvValue":"","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"PG_PORT","EnvType":"string","EnvValue":"5432","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"PG_QUERY_DUR_THRESHOLD","EnvType":"int64","EnvValue":"5000","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"PG_USER","EnvType":"string","EnvValue":"","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"POLL_DURATION","EnvType":"int","EnvValue":"2","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"POLL_WORKER","EnvType":"int","EnvValue":"5","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"SERVER_GRPC_PORT","EnvType":"int","EnvValue":"8081","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"SERVER_REST_PORT","EnvType":"int","EnvValue":"8080","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"STREAM_CONFIG_JSON","EnvType":"string","EnvValue":"","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"USE_GIT_CLI","EnvType":"bool","EnvValue":"false","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"USE_GIT_CLI_ANALYTICS","EnvType":"bool","EnvValue":"false","EnvDescription":"","Example":"","Deprecated":"false"}]}] \ No newline at end of file diff --git a/git-sensor/env_gen.md b/git-sensor/env_gen.md new file mode 100644 index 000000000..b43f8137d --- /dev/null +++ b/git-sensor/env_gen.md @@ -0,0 +1,42 @@ + + +## DEVTRON Related Environment Variables +| Key | Type | Default Value | Description | Example | Deprecated | +|-------|----------|-------------------|-------------------|-----------------------|------------------| + | ANALYTICS_DEBUG | bool |false | | | false | + | APP | string |git-sensor | | | false | + | CLI_CMD_TIMEOUT_GLOBAL_SECONDS | int |900 | | | false | + | CLI_CMD_TIMEOUT_JSON | string | | | | false | + | COMMIT_STATS_TIMEOUT_IN_SEC | int |2 | | | false | + | CONSUMER_CONFIG_JSON | string | | | | false | + | DEFAULT_LOG_TIME_LIMIT | int64 |1 | | | false | + | ENABLE_FILE_STATS | bool |false | | | false | + | ENABLE_STATSVIZ | bool |false | | | false | + | GIT_HISTORY_COUNT | int |15 | | | false | + | GOGIT_TIMEOUT_SECONDS | int |10 | | | false | + | LOG_LEVEL | int |0 | | | false | + | MIN_LIMIT_FOR_PVC | int |1 | | | false | + | NATS_MSG_ACK_WAIT_IN_SECS | int |120 | | | false | + | NATS_MSG_BUFFER_SIZE | int |-1 | | | false | + | NATS_MSG_MAX_AGE | int |86400 | | | false | + | NATS_MSG_PROCESSING_BATCH_SIZE | int |1 | | | false | + | NATS_MSG_REPLICAS | int |0 | | | false | + | NATS_SERVER_HOST | string |nats://devtron-nats.devtroncd:4222 | | | false | + | PG_ADDR | string |127.0.0.1 | | | false | + | PG_DATABASE | string |git_sensor | | | false | + | PG_EXPORT_PROM_METRICS | bool |true | | | false | + | PG_LOG_ALL_FAILURE_QUERIES | bool |true | | | false | + | PG_LOG_ALL_QUERY | bool |false | | | false | + | PG_LOG_SLOW_QUERY | bool |true | | | false | + | PG_PASSWORD | string | | | | false | + | PG_PORT | string |5432 | | | false | + | PG_QUERY_DUR_THRESHOLD | int64 |5000 | | | false | + | PG_USER | string | | | | false | + | POLL_DURATION | int |2 | | | false | + | POLL_WORKER | int |5 | | | false | + | SERVER_GRPC_PORT | int |8081 | | | false | + | SERVER_REST_PORT | int |8080 | | | false | + | STREAM_CONFIG_JSON | string | | | | false | + | USE_GIT_CLI | bool |false | | | false | + | USE_GIT_CLI_ANALYTICS | bool |false | | | false | + diff --git a/git-sensor/fetchAllEnv/fetchAllEnv.go b/git-sensor/fetchAllEnv/fetchAllEnv.go new file mode 100644 index 000000000..ed6163dc6 --- /dev/null +++ b/git-sensor/fetchAllEnv/fetchAllEnv.go @@ -0,0 +1,25 @@ +/* + * Copyright (c) 2024. Devtron Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package main + +import ( + "github.com/devtron-labs/common-lib/fetchAllEnv" +) + +func main() { + fetchAllEnv.FetchEnvAndWriteToFile() +} diff --git a/git-sensor/go.mod b/git-sensor/go.mod index 2d10405de..dd3fbf033 100644 --- a/git-sensor/go.mod +++ b/git-sensor/go.mod @@ -4,7 +4,7 @@ go 1.21 toolchain go1.22.4 -replace github.com/devtron-labs/common-lib => github.com/devtron-labs/devtron-services/common-lib v0.0.0-20250127074233-945aa5b576dd +replace github.com/devtron-labs/common-lib => github.com/devtron-labs/devtron-services/common-lib v0.0.0-20250127103321-97c55f5baa57 require ( github.com/caarlos0/env v3.5.0+incompatible diff --git a/git-sensor/go.sum b/git-sensor/go.sum index c46451006..b017934bf 100644 --- a/git-sensor/go.sum +++ b/git-sensor/go.sum @@ -29,8 +29,8 @@ github.com/cyphar/filepath-securejoin v0.2.4/go.mod h1:aPGpWjXOXUn2NCNjFvBE6aRxG github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= -github.com/devtron-labs/devtron-services/common-lib v0.0.0-20250127074233-945aa5b576dd h1:Mq9sBScfBWU4FWnfihutOXQbvy3t63tViYN9DVrPExw= -github.com/devtron-labs/devtron-services/common-lib v0.0.0-20250127074233-945aa5b576dd/go.mod h1:1QJJLpgJSkb5Jm9xPeKAk+kXb0QgBOOOgJj0cgYhAVA= +github.com/devtron-labs/devtron-services/common-lib v0.0.0-20250127103321-97c55f5baa57 h1:EPMILz/Lk4oK/gTJWiOL5a7jFAiuiokeFFc/T7qeQ/o= +github.com/devtron-labs/devtron-services/common-lib v0.0.0-20250127103321-97c55f5baa57/go.mod h1:1QJJLpgJSkb5Jm9xPeKAk+kXb0QgBOOOgJj0cgYhAVA= github.com/devtron-labs/protos v0.0.3-0.20240809072909-83171af34169 h1:9OMZv0/fOWKK9s9BLTofFL/BO79TdyvC1Sc1HsC4esQ= github.com/devtron-labs/protos v0.0.3-0.20240809072909-83171af34169/go.mod h1:1TqULGlTey+VNhAu/ag7NJuUvByJemkqodsc9L5PHJk= github.com/docker/cli v24.0.6+incompatible h1:fF+XCQCgJjjQNIMjzaSmiKJSCcfcXb3TWTcc7GAneOY= diff --git a/git-sensor/vendor/github.com/devtron-labs/common-lib/fetchAllEnv/fetchAllEnv.go b/git-sensor/vendor/github.com/devtron-labs/common-lib/fetchAllEnv/fetchAllEnv.go new file mode 100644 index 000000000..c8a19e282 --- /dev/null +++ b/git-sensor/vendor/github.com/devtron-labs/common-lib/fetchAllEnv/fetchAllEnv.go @@ -0,0 +1,226 @@ +/* + * Copyright (c) 2024. Devtron Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package fetchAllEnv + +import ( + "encoding/json" + "errors" + "go/ast" + "go/parser" + "go/token" + "log" + "os" + "path/filepath" + "reflect" + "sort" + "strings" + "text/template" +) + +type EnvField struct { + Env string + EnvType string + EnvValue string + EnvDescription string + Example string + Deprecated string +} + +type CategoryField struct { + Category string + Fields []EnvField +} + +const ( + categoryCommentStructPrefix = "CATEGORY=" + defaultCategory = "DEVTRON" + deprecatedDefaultValue = "false" + + envFieldTypeTag = "env" + envDefaultFieldTypeTag = "envDefault" + envDescriptionFieldTypeTag = "description" + envPossibleValuesFieldTypeTag = "example" + envDeprecatedFieldTypeTag = "deprecated" + MARKDOWN_FILENAME = "env_gen.md" + MARKDOWN_JSON_FILENAME = "env_gen.json" +) + +const MarkdownTemplate = ` +{{range . }} +## {{ .Category }} Related Environment Variables +| Key | Type | Default Value | Description | Example | Deprecated | +|-------|----------|-------------------|-------------------|-----------------------|------------------| +{{range .Fields }} | {{ .Env }} | {{ .EnvType }} |{{ .EnvValue }} | {{ .EnvDescription }} | {{ .Example }} | {{ .Deprecated }} | +{{end}} +{{end}}` + +func FetchEnvAndWriteToFile() { + WalkThroughProject() + return +} + +func WalkThroughProject() { + categoryFieldsMap := make(map[string][]EnvField) + uniqueKeys := make(map[string]bool) + err := filepath.Walk(".", func(path string, info os.FileInfo, err error) error { + if err != nil { + return err + } + if !info.IsDir() && strings.HasSuffix(path, ".go") { + err = processGoFile(path, categoryFieldsMap, uniqueKeys) + if err != nil { + log.Println("error in processing go file", err) + return err + } + } + return nil + }) + if err != nil { + return + } + writeToFile(categoryFieldsMap) +} + +func processGoFile(filePath string, categoryFieldsMap map[string][]EnvField, uniqueKeys map[string]bool) error { + fset := token.NewFileSet() + node, err := parser.ParseFile(fset, filePath, nil, parser.ParseComments) + if err != nil { + log.Println("error parsing file:", err) + return err + } + ast.Inspect(node, func(n ast.Node) bool { + if genDecl, ok := n.(*ast.GenDecl); ok { + // checking if type declaration, one of [func, map, struct, array, channel, interface] + if genDecl.Tok == token.TYPE { + for _, spec := range genDecl.Specs { + if typeSpec, ok := spec.(*ast.TypeSpec); ok { + // only checking struct type declarations + if structType, ok2 := typeSpec.Type.(*ast.StructType); ok2 { + allFields := make([]EnvField, 0, len(structType.Fields.List)) + for _, field := range structType.Fields.List { + if field.Tag != nil { + envField := getEnvKeyAndValue(field) + envKey := envField.Env + if len(envKey) == 0 || uniqueKeys[envKey] { + continue + } + allFields = append(allFields, envField) + uniqueKeys[envKey] = true + } + } + if len(allFields) > 0 { + category := getCategoryForAStruct(genDecl) + categoryFieldsMap[category] = append(categoryFieldsMap[category], allFields...) + } + } + } + } + } + } + return true + }) + return nil +} + +func getEnvKeyAndValue(field *ast.Field) EnvField { + tag := reflect.StructTag(strings.Trim(field.Tag.Value, "`")) // remove surrounding backticks + + envKey := addReadmeTableDelimiterEscapeChar(tag.Get(envFieldTypeTag)) + envValue := addReadmeTableDelimiterEscapeChar(tag.Get(envDefaultFieldTypeTag)) + envDescription := addReadmeTableDelimiterEscapeChar(tag.Get(envDescriptionFieldTypeTag)) + envPossibleValues := addReadmeTableDelimiterEscapeChar(tag.Get(envPossibleValuesFieldTypeTag)) + envDeprecated := addReadmeTableDelimiterEscapeChar(tag.Get(envDeprecatedFieldTypeTag)) + // check if there exist any value provided in env for this field + if value, ok := os.LookupEnv(envKey); ok { + envValue = value + } + env := EnvField{ + Env: envKey, + EnvValue: envValue, + EnvDescription: envDescription, + Example: envPossibleValues, + Deprecated: envDeprecated, + } + if indent, ok := field.Type.(*ast.Ident); ok && indent != nil { + env.EnvType = indent.Name + } + if len(envDeprecated) == 0 { + env.Deprecated = deprecatedDefaultValue + } + return env +} + +func getCategoryForAStruct(genDecl *ast.GenDecl) string { + category := defaultCategory + if genDecl.Doc != nil { + commentTexts := strings.Split(genDecl.Doc.Text(), "\n") + for _, comment := range commentTexts { + commentText := strings.TrimPrefix(strings.ReplaceAll(comment, " ", ""), "//") // this can happen if comment group is in /* */ + if strings.HasPrefix(commentText, categoryCommentStructPrefix) { + categories := strings.Split(strings.TrimPrefix(commentText, categoryCommentStructPrefix), ",") + if len(categories) > 0 && len(categories[0]) > 0 { //only supporting one category as of now + category = categories[0] //overriding category + break + } + } + } + } + return category +} + +func addReadmeTableDelimiterEscapeChar(s string) string { + return strings.ReplaceAll(s, "|", `\|`) +} + +func writeToFile(categoryFieldsMap map[string][]EnvField) { + cfs := make([]CategoryField, 0, len(categoryFieldsMap)) + for category, allFields := range categoryFieldsMap { + sort.Slice(allFields, func(i, j int) bool { + return allFields[i].Env < allFields[j].Env + }) + + cfs = append(cfs, CategoryField{ + Category: category, + Fields: allFields, + }) + } + sort.Slice(cfs, func(i, j int) bool { + return cfs[i].Category < cfs[j].Category + }) + file, err := os.Create(MARKDOWN_FILENAME) + if err != nil && !errors.Is(err, os.ErrExist) { + panic(err) + } + defer file.Close() + tmpl, err := template.New("markdown").Parse(MarkdownTemplate) + if err != nil { + panic(err) + } + err = tmpl.Execute(file, cfs) + if err != nil { + panic(err) + } + cfsMarshaled, err := json.Marshal(cfs) + if err != nil { + log.Println("error marshalling category fields:", err) + panic(err) + } + err = os.WriteFile(MARKDOWN_JSON_FILENAME, cfsMarshaled, 0644) + if err != nil { + panic(err) + } +} diff --git a/git-sensor/vendor/modules.txt b/git-sensor/vendor/modules.txt index f51c774b6..7f7902741 100644 --- a/git-sensor/vendor/modules.txt +++ b/git-sensor/vendor/modules.txt @@ -62,9 +62,10 @@ github.com/cyphar/filepath-securejoin # github.com/davecgh/go-spew v1.1.1 ## explicit github.com/davecgh/go-spew/spew -# github.com/devtron-labs/common-lib v0.0.0 => github.com/devtron-labs/devtron-services/common-lib v0.0.0-20250127074233-945aa5b576dd +# github.com/devtron-labs/common-lib v0.0.0 => github.com/devtron-labs/devtron-services/common-lib v0.0.0-20250127103321-97c55f5baa57 ## explicit; go 1.21 github.com/devtron-labs/common-lib/constants +github.com/devtron-labs/common-lib/fetchAllEnv github.com/devtron-labs/common-lib/git-manager github.com/devtron-labs/common-lib/git-manager/util github.com/devtron-labs/common-lib/middlewares @@ -481,4 +482,4 @@ gopkg.in/yaml.v3 # mellium.im/sasl v0.3.2 ## explicit; go 1.20 mellium.im/sasl -# github.com/devtron-labs/common-lib => github.com/devtron-labs/devtron-services/common-lib v0.0.0-20250127074233-945aa5b576dd +# github.com/devtron-labs/common-lib => github.com/devtron-labs/devtron-services/common-lib v0.0.0-20250127103321-97c55f5baa57 diff --git a/image-scanner/Makefile b/image-scanner/Makefile index fdee91c23..b701ddf69 100644 --- a/image-scanner/Makefile +++ b/image-scanner/Makefile @@ -1,5 +1,5 @@ -all: build +all: fetch-all-env build TAG?=latest FLAGS= @@ -31,4 +31,7 @@ docker-build-image: build dep-update-oss: go mod edit -replace=github.com/devtron-labs/common-lib=github.com/devtron-labs/devtron-services/common-lib@$(TARGET_BRANCH) go mod tidy - go mod vendor \ No newline at end of file + go mod vendor + +fetch-all-env: + go run fetchAllEnv/fetchAllEnv.go diff --git a/image-scanner/env_gen.json b/image-scanner/env_gen.json new file mode 100644 index 000000000..1d5fbcfc1 --- /dev/null +++ b/image-scanner/env_gen.json @@ -0,0 +1 @@ +[{"Category":"DEVTRON","Fields":[{"Env":"APP","EnvType":"string","EnvValue":"image-scanner","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"CLAIR_ADDR","EnvType":"string","EnvValue":"http://localhost:6060","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"CLAIR_TIMEOUT","EnvType":"int","EnvValue":"30","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"CONSUMER_CONFIG_JSON","EnvType":"string","EnvValue":"","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"DEFAULT_LOG_TIME_LIMIT","EnvType":"int64","EnvValue":"1","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"ENABLE_STATSVIZ","EnvType":"bool","EnvValue":"false","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"IMAGE_SCAN_ASYNC_TIMEOUT","EnvType":"int","EnvValue":"3","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"IMAGE_SCAN_TIMEOUT","EnvType":"int","EnvValue":"10","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"IMAGE_SCAN_TRY_COUNT","EnvType":"int","EnvValue":"1","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"JSON_OUTPUT","EnvType":"bool","EnvValue":"true","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"LOG_LEVEL","EnvType":"int","EnvValue":"0","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"NATS_MSG_ACK_WAIT_IN_SECS","EnvType":"int","EnvValue":"120","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"NATS_MSG_BUFFER_SIZE","EnvType":"int","EnvValue":"-1","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"NATS_MSG_MAX_AGE","EnvType":"int","EnvValue":"86400","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"NATS_MSG_PROCESSING_BATCH_SIZE","EnvType":"int","EnvValue":"1","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"NATS_MSG_REPLICAS","EnvType":"int","EnvValue":"0","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"NATS_SERVER_HOST","EnvType":"string","EnvValue":"nats://devtron-nats.devtroncd:4222","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"PG_ADDR","EnvType":"string","EnvValue":"127.0.0.1","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"PG_DATABASE","EnvType":"string","EnvValue":"orchestrator","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"PG_EXPORT_PROM_METRICS","EnvType":"bool","EnvValue":"true","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"PG_LOG_ALL_FAILURE_QUERIES","EnvType":"bool","EnvValue":"true","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"PG_LOG_ALL_QUERY","EnvType":"bool","EnvValue":"false","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"PG_LOG_SLOW_QUERY","EnvType":"bool","EnvValue":"true","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"PG_PASSWORD","EnvType":"string","EnvValue":"","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"PG_PORT","EnvType":"string","EnvValue":"5432","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"PG_QUERY_DUR_THRESHOLD","EnvType":"int64","EnvValue":"5000","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"PG_USER","EnvType":"string","EnvValue":"","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"PROJECT_ID","EnvType":"string","EnvValue":"projects/devtron-project-id","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"SCANNER_TYPE","EnvType":"string","EnvValue":"","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"SERVER_HTTP_PORT","EnvType":"int","EnvValue":"8080","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"STREAM_CONFIG_JSON","EnvType":"string","EnvValue":"","EnvDescription":"","Example":"","Deprecated":"false"}]}] \ No newline at end of file diff --git a/image-scanner/env_gen.md b/image-scanner/env_gen.md new file mode 100644 index 000000000..7d8c12043 --- /dev/null +++ b/image-scanner/env_gen.md @@ -0,0 +1,37 @@ + + +## DEVTRON Related Environment Variables +| Key | Type | Default Value | Description | Example | Deprecated | +|-------|----------|-------------------|-------------------|-----------------------|------------------| + | APP | string |image-scanner | | | false | + | CLAIR_ADDR | string |http://localhost:6060 | | | false | + | CLAIR_TIMEOUT | int |30 | | | false | + | CONSUMER_CONFIG_JSON | string | | | | false | + | DEFAULT_LOG_TIME_LIMIT | int64 |1 | | | false | + | ENABLE_STATSVIZ | bool |false | | | false | + | IMAGE_SCAN_ASYNC_TIMEOUT | int |3 | | | false | + | IMAGE_SCAN_TIMEOUT | int |10 | | | false | + | IMAGE_SCAN_TRY_COUNT | int |1 | | | false | + | JSON_OUTPUT | bool |true | | | false | + | LOG_LEVEL | int |0 | | | false | + | NATS_MSG_ACK_WAIT_IN_SECS | int |120 | | | false | + | NATS_MSG_BUFFER_SIZE | int |-1 | | | false | + | NATS_MSG_MAX_AGE | int |86400 | | | false | + | NATS_MSG_PROCESSING_BATCH_SIZE | int |1 | | | false | + | NATS_MSG_REPLICAS | int |0 | | | false | + | NATS_SERVER_HOST | string |nats://devtron-nats.devtroncd:4222 | | | false | + | PG_ADDR | string |127.0.0.1 | | | false | + | PG_DATABASE | string |orchestrator | | | false | + | PG_EXPORT_PROM_METRICS | bool |true | | | false | + | PG_LOG_ALL_FAILURE_QUERIES | bool |true | | | false | + | PG_LOG_ALL_QUERY | bool |false | | | false | + | PG_LOG_SLOW_QUERY | bool |true | | | false | + | PG_PASSWORD | string | | | | false | + | PG_PORT | string |5432 | | | false | + | PG_QUERY_DUR_THRESHOLD | int64 |5000 | | | false | + | PG_USER | string | | | | false | + | PROJECT_ID | string |projects/devtron-project-id | | | false | + | SCANNER_TYPE | string | | | | false | + | SERVER_HTTP_PORT | int |8080 | | | false | + | STREAM_CONFIG_JSON | string | | | | false | + diff --git a/image-scanner/fetchAllEnv/fetchAllEnv.go b/image-scanner/fetchAllEnv/fetchAllEnv.go new file mode 100644 index 000000000..ed6163dc6 --- /dev/null +++ b/image-scanner/fetchAllEnv/fetchAllEnv.go @@ -0,0 +1,25 @@ +/* + * Copyright (c) 2024. Devtron Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package main + +import ( + "github.com/devtron-labs/common-lib/fetchAllEnv" +) + +func main() { + fetchAllEnv.FetchEnvAndWriteToFile() +} diff --git a/image-scanner/go.mod b/image-scanner/go.mod index 02b7bd9ac..5e1b5b914 100644 --- a/image-scanner/go.mod +++ b/image-scanner/go.mod @@ -74,4 +74,4 @@ require ( mellium.im/sasl v0.3.2 // indirect ) -replace github.com/devtron-labs/common-lib => github.com/devtron-labs/devtron-services/common-lib v0.0.0-20250127074233-945aa5b576dd +replace github.com/devtron-labs/common-lib => github.com/devtron-labs/devtron-services/common-lib v0.0.0-20250127103321-97c55f5baa57 diff --git a/image-scanner/go.sum b/image-scanner/go.sum index b32897f1d..728401d0e 100644 --- a/image-scanner/go.sum +++ b/image-scanner/go.sum @@ -280,8 +280,8 @@ github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSs github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/denverdino/aliyungo v0.0.0-20190125010748-a747050bb1ba/go.mod h1:dV8lFg6daOBZbT6/BDGIz6Y3WFGn8juu6G+CQ6LHtl0= -github.com/devtron-labs/devtron-services/common-lib v0.0.0-20250127074233-945aa5b576dd h1:Mq9sBScfBWU4FWnfihutOXQbvy3t63tViYN9DVrPExw= -github.com/devtron-labs/devtron-services/common-lib v0.0.0-20250127074233-945aa5b576dd/go.mod h1:1QJJLpgJSkb5Jm9xPeKAk+kXb0QgBOOOgJj0cgYhAVA= +github.com/devtron-labs/devtron-services/common-lib v0.0.0-20250127103321-97c55f5baa57 h1:EPMILz/Lk4oK/gTJWiOL5a7jFAiuiokeFFc/T7qeQ/o= +github.com/devtron-labs/devtron-services/common-lib v0.0.0-20250127103321-97c55f5baa57/go.mod h1:1QJJLpgJSkb5Jm9xPeKAk+kXb0QgBOOOgJj0cgYhAVA= github.com/dgrijalva/jwt-go v0.0.0-20170104182250-a601269ab70c/go.mod h1:E3ru+11k8xSBh+hMPgOLZmtrrCbhqsmaPHjLKYnJCaQ= github.com/dgrijalva/jwt-go v3.2.0+incompatible/go.mod h1:E3ru+11k8xSBh+hMPgOLZmtrrCbhqsmaPHjLKYnJCaQ= github.com/dgryski/go-sip13 v0.0.0-20181026042036-e10d5fee7954/go.mod h1:vAd38F8PWV+bWy6jNmig1y/TA+kYO4g3RSRF0IAv0no= diff --git a/image-scanner/vendor/github.com/devtron-labs/common-lib/fetchAllEnv/fetchAllEnv.go b/image-scanner/vendor/github.com/devtron-labs/common-lib/fetchAllEnv/fetchAllEnv.go new file mode 100644 index 000000000..c8a19e282 --- /dev/null +++ b/image-scanner/vendor/github.com/devtron-labs/common-lib/fetchAllEnv/fetchAllEnv.go @@ -0,0 +1,226 @@ +/* + * Copyright (c) 2024. Devtron Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package fetchAllEnv + +import ( + "encoding/json" + "errors" + "go/ast" + "go/parser" + "go/token" + "log" + "os" + "path/filepath" + "reflect" + "sort" + "strings" + "text/template" +) + +type EnvField struct { + Env string + EnvType string + EnvValue string + EnvDescription string + Example string + Deprecated string +} + +type CategoryField struct { + Category string + Fields []EnvField +} + +const ( + categoryCommentStructPrefix = "CATEGORY=" + defaultCategory = "DEVTRON" + deprecatedDefaultValue = "false" + + envFieldTypeTag = "env" + envDefaultFieldTypeTag = "envDefault" + envDescriptionFieldTypeTag = "description" + envPossibleValuesFieldTypeTag = "example" + envDeprecatedFieldTypeTag = "deprecated" + MARKDOWN_FILENAME = "env_gen.md" + MARKDOWN_JSON_FILENAME = "env_gen.json" +) + +const MarkdownTemplate = ` +{{range . }} +## {{ .Category }} Related Environment Variables +| Key | Type | Default Value | Description | Example | Deprecated | +|-------|----------|-------------------|-------------------|-----------------------|------------------| +{{range .Fields }} | {{ .Env }} | {{ .EnvType }} |{{ .EnvValue }} | {{ .EnvDescription }} | {{ .Example }} | {{ .Deprecated }} | +{{end}} +{{end}}` + +func FetchEnvAndWriteToFile() { + WalkThroughProject() + return +} + +func WalkThroughProject() { + categoryFieldsMap := make(map[string][]EnvField) + uniqueKeys := make(map[string]bool) + err := filepath.Walk(".", func(path string, info os.FileInfo, err error) error { + if err != nil { + return err + } + if !info.IsDir() && strings.HasSuffix(path, ".go") { + err = processGoFile(path, categoryFieldsMap, uniqueKeys) + if err != nil { + log.Println("error in processing go file", err) + return err + } + } + return nil + }) + if err != nil { + return + } + writeToFile(categoryFieldsMap) +} + +func processGoFile(filePath string, categoryFieldsMap map[string][]EnvField, uniqueKeys map[string]bool) error { + fset := token.NewFileSet() + node, err := parser.ParseFile(fset, filePath, nil, parser.ParseComments) + if err != nil { + log.Println("error parsing file:", err) + return err + } + ast.Inspect(node, func(n ast.Node) bool { + if genDecl, ok := n.(*ast.GenDecl); ok { + // checking if type declaration, one of [func, map, struct, array, channel, interface] + if genDecl.Tok == token.TYPE { + for _, spec := range genDecl.Specs { + if typeSpec, ok := spec.(*ast.TypeSpec); ok { + // only checking struct type declarations + if structType, ok2 := typeSpec.Type.(*ast.StructType); ok2 { + allFields := make([]EnvField, 0, len(structType.Fields.List)) + for _, field := range structType.Fields.List { + if field.Tag != nil { + envField := getEnvKeyAndValue(field) + envKey := envField.Env + if len(envKey) == 0 || uniqueKeys[envKey] { + continue + } + allFields = append(allFields, envField) + uniqueKeys[envKey] = true + } + } + if len(allFields) > 0 { + category := getCategoryForAStruct(genDecl) + categoryFieldsMap[category] = append(categoryFieldsMap[category], allFields...) + } + } + } + } + } + } + return true + }) + return nil +} + +func getEnvKeyAndValue(field *ast.Field) EnvField { + tag := reflect.StructTag(strings.Trim(field.Tag.Value, "`")) // remove surrounding backticks + + envKey := addReadmeTableDelimiterEscapeChar(tag.Get(envFieldTypeTag)) + envValue := addReadmeTableDelimiterEscapeChar(tag.Get(envDefaultFieldTypeTag)) + envDescription := addReadmeTableDelimiterEscapeChar(tag.Get(envDescriptionFieldTypeTag)) + envPossibleValues := addReadmeTableDelimiterEscapeChar(tag.Get(envPossibleValuesFieldTypeTag)) + envDeprecated := addReadmeTableDelimiterEscapeChar(tag.Get(envDeprecatedFieldTypeTag)) + // check if there exist any value provided in env for this field + if value, ok := os.LookupEnv(envKey); ok { + envValue = value + } + env := EnvField{ + Env: envKey, + EnvValue: envValue, + EnvDescription: envDescription, + Example: envPossibleValues, + Deprecated: envDeprecated, + } + if indent, ok := field.Type.(*ast.Ident); ok && indent != nil { + env.EnvType = indent.Name + } + if len(envDeprecated) == 0 { + env.Deprecated = deprecatedDefaultValue + } + return env +} + +func getCategoryForAStruct(genDecl *ast.GenDecl) string { + category := defaultCategory + if genDecl.Doc != nil { + commentTexts := strings.Split(genDecl.Doc.Text(), "\n") + for _, comment := range commentTexts { + commentText := strings.TrimPrefix(strings.ReplaceAll(comment, " ", ""), "//") // this can happen if comment group is in /* */ + if strings.HasPrefix(commentText, categoryCommentStructPrefix) { + categories := strings.Split(strings.TrimPrefix(commentText, categoryCommentStructPrefix), ",") + if len(categories) > 0 && len(categories[0]) > 0 { //only supporting one category as of now + category = categories[0] //overriding category + break + } + } + } + } + return category +} + +func addReadmeTableDelimiterEscapeChar(s string) string { + return strings.ReplaceAll(s, "|", `\|`) +} + +func writeToFile(categoryFieldsMap map[string][]EnvField) { + cfs := make([]CategoryField, 0, len(categoryFieldsMap)) + for category, allFields := range categoryFieldsMap { + sort.Slice(allFields, func(i, j int) bool { + return allFields[i].Env < allFields[j].Env + }) + + cfs = append(cfs, CategoryField{ + Category: category, + Fields: allFields, + }) + } + sort.Slice(cfs, func(i, j int) bool { + return cfs[i].Category < cfs[j].Category + }) + file, err := os.Create(MARKDOWN_FILENAME) + if err != nil && !errors.Is(err, os.ErrExist) { + panic(err) + } + defer file.Close() + tmpl, err := template.New("markdown").Parse(MarkdownTemplate) + if err != nil { + panic(err) + } + err = tmpl.Execute(file, cfs) + if err != nil { + panic(err) + } + cfsMarshaled, err := json.Marshal(cfs) + if err != nil { + log.Println("error marshalling category fields:", err) + panic(err) + } + err = os.WriteFile(MARKDOWN_JSON_FILENAME, cfsMarshaled, 0644) + if err != nil { + panic(err) + } +} diff --git a/image-scanner/vendor/modules.txt b/image-scanner/vendor/modules.txt index d9ffec318..cee9e17af 100644 --- a/image-scanner/vendor/modules.txt +++ b/image-scanner/vendor/modules.txt @@ -72,9 +72,10 @@ github.com/cespare/xxhash/v2 github.com/coreos/clair/api/v3/clairpb github.com/coreos/clair/database github.com/coreos/clair/ext/versionfmt -# github.com/devtron-labs/common-lib v0.19.0 => github.com/devtron-labs/devtron-services/common-lib v0.0.0-20250127074233-945aa5b576dd +# github.com/devtron-labs/common-lib v0.19.0 => github.com/devtron-labs/devtron-services/common-lib v0.0.0-20250127103321-97c55f5baa57 ## explicit; go 1.21 github.com/devtron-labs/common-lib/constants +github.com/devtron-labs/common-lib/fetchAllEnv github.com/devtron-labs/common-lib/git-manager github.com/devtron-labs/common-lib/git-manager/util github.com/devtron-labs/common-lib/imageScan/bean @@ -439,4 +440,4 @@ google.golang.org/protobuf/types/known/wrapperspb # mellium.im/sasl v0.3.2 ## explicit; go 1.20 mellium.im/sasl -# github.com/devtron-labs/common-lib => github.com/devtron-labs/devtron-services/common-lib v0.0.0-20250127074233-945aa5b576dd +# github.com/devtron-labs/common-lib => github.com/devtron-labs/devtron-services/common-lib v0.0.0-20250127103321-97c55f5baa57 diff --git a/image-scanner/wire_gen.go b/image-scanner/wire_gen.go index b668a2a40..d3a947ce0 100644 --- a/image-scanner/wire_gen.go +++ b/image-scanner/wire_gen.go @@ -1,6 +1,6 @@ // Code generated by Wire. DO NOT EDIT. -//go:generate go run github.com/google/wire/cmd/wire +//go:generate go run -mod=mod github.com/google/wire/cmd/wire //go:build !wireinject // +build !wireinject diff --git a/kubelink/Makefile b/kubelink/Makefile index f4ca41fac..cbe15df31 100644 --- a/kubelink/Makefile +++ b/kubelink/Makefile @@ -1,5 +1,5 @@ -all: build +all: fetch-all-env build TAG?=latest FLAGS= @@ -32,3 +32,6 @@ dep-update-oss: go mod tidy go mod vendor + +fetch-all-env: + go run fetchAllEnv/fetchAllEnv.go diff --git a/kubelink/env_gen.json b/kubelink/env_gen.json new file mode 100644 index 000000000..254a663e8 --- /dev/null +++ b/kubelink/env_gen.json @@ -0,0 +1 @@ +[{"Category":"DEVTRON","Fields":[{"Env":"APP","EnvType":"string","EnvValue":"kubelink","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"BUILD_NODES_BATCH_SIZE","EnvType":"int","EnvValue":"2","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"CHART_WORKING_DIRECTORY","EnvType":"string","EnvValue":"/home/devtron/devtroncd/charts/","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"CONSUMER_CONFIG_JSON","EnvType":"string","EnvValue":"","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"DEFAULT_LOG_TIME_LIMIT","EnvType":"int64","EnvValue":"1","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"ENABLE_HELM_RELEASE_CACHE","EnvType":"bool","EnvValue":"true","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"ENABLE_STATSVIZ","EnvType":"bool","EnvValue":"false","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"K8s_CLIENT_MAX_IDLE_CONNS_PER_HOST","EnvType":"int","EnvValue":"25","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"K8s_TCP_IDLE_CONN_TIMEOUT","EnvType":"int","EnvValue":"300","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"K8s_TCP_KEEPALIVE","EnvType":"int","EnvValue":"30","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"K8s_TCP_TIMEOUT","EnvType":"int","EnvValue":"30","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"K8s_TLS_HANDSHAKE_TIMEOUT","EnvType":"int","EnvValue":"10","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"KUBELINK_GRPC_MAX_RECEIVE_MSG_SIZE","EnvType":"int","EnvValue":"20","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"KUBELINK_GRPC_MAX_SEND_MSG_SIZE","EnvType":"int","EnvValue":"4","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"LOG_LEVEL","EnvType":"int","EnvValue":"-1","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"MANIFEST_FETCH_BATCH_SIZE","EnvType":"int","EnvValue":"2","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"MAX_COUNT_FOR_HELM_RELEASE","EnvType":"int","EnvValue":"20","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"NATS_MSG_ACK_WAIT_IN_SECS","EnvType":"int","EnvValue":"120","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"NATS_MSG_BUFFER_SIZE","EnvType":"int","EnvValue":"-1","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"NATS_MSG_MAX_AGE","EnvType":"int","EnvValue":"86400","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"NATS_MSG_PROCESSING_BATCH_SIZE","EnvType":"int","EnvValue":"1","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"NATS_MSG_REPLICAS","EnvType":"int","EnvValue":"0","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"NATS_SERVER_HOST","EnvType":"string","EnvValue":"nats://devtron-nats.devtroncd:4222","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"PARENT_CHILD_GVK_MAPPING","EnvType":"string","EnvValue":"","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"PG_ADDR","EnvType":"string","EnvValue":"127.0.0.1","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"PG_DATABASE","EnvType":"string","EnvValue":"orchestrator","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"PG_EXPORT_PROM_METRICS","EnvType":"bool","EnvValue":"true","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"PG_LOG_ALL_FAILURE_QUERIES","EnvType":"bool","EnvValue":"true","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"PG_LOG_ALL_QUERY","EnvType":"bool","EnvValue":"false","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"PG_LOG_SLOW_QUERY","EnvType":"bool","EnvValue":"true","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"PG_PASSWORD","EnvType":"string","EnvValue":"","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"PG_PORT","EnvType":"string","EnvValue":"5432","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"PG_QUERY_DUR_THRESHOLD","EnvType":"int64","EnvValue":"5000","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"PG_USER","EnvType":"string","EnvValue":"","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"RUNTIME_CONFIG_LOCAL_DEV","EnvType":"LocalDevMode","EnvValue":"false","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"RUN_HELM_INSTALL_IN_ASYNC_MODE","EnvType":"bool","EnvValue":"false","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"STREAM_CONFIG_JSON","EnvType":"string","EnvValue":"","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"USE_CUSTOM_HTTP_TRANSPORT","EnvType":"bool","EnvValue":"false","EnvDescription":"","Example":"","Deprecated":"false"}]}] \ No newline at end of file diff --git a/kubelink/env_gen.md b/kubelink/env_gen.md new file mode 100644 index 000000000..517fdf8d9 --- /dev/null +++ b/kubelink/env_gen.md @@ -0,0 +1,44 @@ + + +## DEVTRON Related Environment Variables +| Key | Type | Default Value | Description | Example | Deprecated | +|-------|----------|-------------------|-------------------|-----------------------|------------------| + | APP | string |kubelink | | | false | + | BUILD_NODES_BATCH_SIZE | int |2 | | | false | + | CHART_WORKING_DIRECTORY | string |/home/devtron/devtroncd/charts/ | | | false | + | CONSUMER_CONFIG_JSON | string | | | | false | + | DEFAULT_LOG_TIME_LIMIT | int64 |1 | | | false | + | ENABLE_HELM_RELEASE_CACHE | bool |true | | | false | + | ENABLE_STATSVIZ | bool |false | | | false | + | K8s_CLIENT_MAX_IDLE_CONNS_PER_HOST | int |25 | | | false | + | K8s_TCP_IDLE_CONN_TIMEOUT | int |300 | | | false | + | K8s_TCP_KEEPALIVE | int |30 | | | false | + | K8s_TCP_TIMEOUT | int |30 | | | false | + | K8s_TLS_HANDSHAKE_TIMEOUT | int |10 | | | false | + | KUBELINK_GRPC_MAX_RECEIVE_MSG_SIZE | int |20 | | | false | + | KUBELINK_GRPC_MAX_SEND_MSG_SIZE | int |4 | | | false | + | LOG_LEVEL | int |-1 | | | false | + | MANIFEST_FETCH_BATCH_SIZE | int |2 | | | false | + | MAX_COUNT_FOR_HELM_RELEASE | int |20 | | | false | + | NATS_MSG_ACK_WAIT_IN_SECS | int |120 | | | false | + | NATS_MSG_BUFFER_SIZE | int |-1 | | | false | + | NATS_MSG_MAX_AGE | int |86400 | | | false | + | NATS_MSG_PROCESSING_BATCH_SIZE | int |1 | | | false | + | NATS_MSG_REPLICAS | int |0 | | | false | + | NATS_SERVER_HOST | string |nats://devtron-nats.devtroncd:4222 | | | false | + | PARENT_CHILD_GVK_MAPPING | string | | | | false | + | PG_ADDR | string |127.0.0.1 | | | false | + | PG_DATABASE | string |orchestrator | | | false | + | PG_EXPORT_PROM_METRICS | bool |true | | | false | + | PG_LOG_ALL_FAILURE_QUERIES | bool |true | | | false | + | PG_LOG_ALL_QUERY | bool |false | | | false | + | PG_LOG_SLOW_QUERY | bool |true | | | false | + | PG_PASSWORD | string | | | | false | + | PG_PORT | string |5432 | | | false | + | PG_QUERY_DUR_THRESHOLD | int64 |5000 | | | false | + | PG_USER | string | | | | false | + | RUNTIME_CONFIG_LOCAL_DEV | LocalDevMode |false | | | false | + | RUN_HELM_INSTALL_IN_ASYNC_MODE | bool |false | | | false | + | STREAM_CONFIG_JSON | string | | | | false | + | USE_CUSTOM_HTTP_TRANSPORT | bool |false | | | false | + diff --git a/kubelink/fetchAllEnv/fetchAllEnv.go b/kubelink/fetchAllEnv/fetchAllEnv.go new file mode 100644 index 000000000..ed6163dc6 --- /dev/null +++ b/kubelink/fetchAllEnv/fetchAllEnv.go @@ -0,0 +1,25 @@ +/* + * Copyright (c) 2024. Devtron Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package main + +import ( + "github.com/devtron-labs/common-lib/fetchAllEnv" +) + +func main() { + fetchAllEnv.FetchEnvAndWriteToFile() +} diff --git a/kubelink/go.mod b/kubelink/go.mod index 3f97b42b0..48e1c5275 100644 --- a/kubelink/go.mod +++ b/kubelink/go.mod @@ -177,7 +177,7 @@ require ( ) replace ( - github.com/devtron-labs/common-lib => github.com/devtron-labs/devtron-services/common-lib v0.0.0-20250127074233-945aa5b576dd + github.com/devtron-labs/common-lib => github.com/devtron-labs/devtron-services/common-lib v0.0.0-20250127103321-97c55f5baa57 go.opentelemetry.io/otel/metric => go.opentelemetry.io/otel/metric v1.18.0 // https://github.com/kubernetes/kubernetes/issues/79384#issuecomment-505627280 k8s.io/api => k8s.io/api v0.29.0 diff --git a/kubelink/go.sum b/kubelink/go.sum index f392f0063..5efadde29 100644 --- a/kubelink/go.sum +++ b/kubelink/go.sum @@ -79,8 +79,8 @@ github.com/cyphar/filepath-securejoin v0.2.4/go.mod h1:aPGpWjXOXUn2NCNjFvBE6aRxG github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= -github.com/devtron-labs/devtron-services/common-lib v0.0.0-20250127074233-945aa5b576dd h1:Mq9sBScfBWU4FWnfihutOXQbvy3t63tViYN9DVrPExw= -github.com/devtron-labs/devtron-services/common-lib v0.0.0-20250127074233-945aa5b576dd/go.mod h1:1QJJLpgJSkb5Jm9xPeKAk+kXb0QgBOOOgJj0cgYhAVA= +github.com/devtron-labs/devtron-services/common-lib v0.0.0-20250127103321-97c55f5baa57 h1:EPMILz/Lk4oK/gTJWiOL5a7jFAiuiokeFFc/T7qeQ/o= +github.com/devtron-labs/devtron-services/common-lib v0.0.0-20250127103321-97c55f5baa57/go.mod h1:1QJJLpgJSkb5Jm9xPeKAk+kXb0QgBOOOgJj0cgYhAVA= github.com/dgryski/go-rendezvous v0.0.0-20200823014737-9f7001d12a5f h1:lO4WD4F/rVNCu3HqELle0jiPLLBs70cWOduZpkS1E78= github.com/dgryski/go-rendezvous v0.0.0-20200823014737-9f7001d12a5f/go.mod h1:cuUVRXasLTGF7a8hSLbxyZXjz+1KgoB3wDUb6vlszIc= github.com/distribution/distribution/v3 v3.0.0-beta.1 h1:X+ELTxPuZ1Xe5MsD3kp2wfGUhc8I+MPfRis8dZ818Ic= diff --git a/kubelink/vendor/github.com/devtron-labs/common-lib/fetchAllEnv/fetchAllEnv.go b/kubelink/vendor/github.com/devtron-labs/common-lib/fetchAllEnv/fetchAllEnv.go new file mode 100644 index 000000000..c8a19e282 --- /dev/null +++ b/kubelink/vendor/github.com/devtron-labs/common-lib/fetchAllEnv/fetchAllEnv.go @@ -0,0 +1,226 @@ +/* + * Copyright (c) 2024. Devtron Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package fetchAllEnv + +import ( + "encoding/json" + "errors" + "go/ast" + "go/parser" + "go/token" + "log" + "os" + "path/filepath" + "reflect" + "sort" + "strings" + "text/template" +) + +type EnvField struct { + Env string + EnvType string + EnvValue string + EnvDescription string + Example string + Deprecated string +} + +type CategoryField struct { + Category string + Fields []EnvField +} + +const ( + categoryCommentStructPrefix = "CATEGORY=" + defaultCategory = "DEVTRON" + deprecatedDefaultValue = "false" + + envFieldTypeTag = "env" + envDefaultFieldTypeTag = "envDefault" + envDescriptionFieldTypeTag = "description" + envPossibleValuesFieldTypeTag = "example" + envDeprecatedFieldTypeTag = "deprecated" + MARKDOWN_FILENAME = "env_gen.md" + MARKDOWN_JSON_FILENAME = "env_gen.json" +) + +const MarkdownTemplate = ` +{{range . }} +## {{ .Category }} Related Environment Variables +| Key | Type | Default Value | Description | Example | Deprecated | +|-------|----------|-------------------|-------------------|-----------------------|------------------| +{{range .Fields }} | {{ .Env }} | {{ .EnvType }} |{{ .EnvValue }} | {{ .EnvDescription }} | {{ .Example }} | {{ .Deprecated }} | +{{end}} +{{end}}` + +func FetchEnvAndWriteToFile() { + WalkThroughProject() + return +} + +func WalkThroughProject() { + categoryFieldsMap := make(map[string][]EnvField) + uniqueKeys := make(map[string]bool) + err := filepath.Walk(".", func(path string, info os.FileInfo, err error) error { + if err != nil { + return err + } + if !info.IsDir() && strings.HasSuffix(path, ".go") { + err = processGoFile(path, categoryFieldsMap, uniqueKeys) + if err != nil { + log.Println("error in processing go file", err) + return err + } + } + return nil + }) + if err != nil { + return + } + writeToFile(categoryFieldsMap) +} + +func processGoFile(filePath string, categoryFieldsMap map[string][]EnvField, uniqueKeys map[string]bool) error { + fset := token.NewFileSet() + node, err := parser.ParseFile(fset, filePath, nil, parser.ParseComments) + if err != nil { + log.Println("error parsing file:", err) + return err + } + ast.Inspect(node, func(n ast.Node) bool { + if genDecl, ok := n.(*ast.GenDecl); ok { + // checking if type declaration, one of [func, map, struct, array, channel, interface] + if genDecl.Tok == token.TYPE { + for _, spec := range genDecl.Specs { + if typeSpec, ok := spec.(*ast.TypeSpec); ok { + // only checking struct type declarations + if structType, ok2 := typeSpec.Type.(*ast.StructType); ok2 { + allFields := make([]EnvField, 0, len(structType.Fields.List)) + for _, field := range structType.Fields.List { + if field.Tag != nil { + envField := getEnvKeyAndValue(field) + envKey := envField.Env + if len(envKey) == 0 || uniqueKeys[envKey] { + continue + } + allFields = append(allFields, envField) + uniqueKeys[envKey] = true + } + } + if len(allFields) > 0 { + category := getCategoryForAStruct(genDecl) + categoryFieldsMap[category] = append(categoryFieldsMap[category], allFields...) + } + } + } + } + } + } + return true + }) + return nil +} + +func getEnvKeyAndValue(field *ast.Field) EnvField { + tag := reflect.StructTag(strings.Trim(field.Tag.Value, "`")) // remove surrounding backticks + + envKey := addReadmeTableDelimiterEscapeChar(tag.Get(envFieldTypeTag)) + envValue := addReadmeTableDelimiterEscapeChar(tag.Get(envDefaultFieldTypeTag)) + envDescription := addReadmeTableDelimiterEscapeChar(tag.Get(envDescriptionFieldTypeTag)) + envPossibleValues := addReadmeTableDelimiterEscapeChar(tag.Get(envPossibleValuesFieldTypeTag)) + envDeprecated := addReadmeTableDelimiterEscapeChar(tag.Get(envDeprecatedFieldTypeTag)) + // check if there exist any value provided in env for this field + if value, ok := os.LookupEnv(envKey); ok { + envValue = value + } + env := EnvField{ + Env: envKey, + EnvValue: envValue, + EnvDescription: envDescription, + Example: envPossibleValues, + Deprecated: envDeprecated, + } + if indent, ok := field.Type.(*ast.Ident); ok && indent != nil { + env.EnvType = indent.Name + } + if len(envDeprecated) == 0 { + env.Deprecated = deprecatedDefaultValue + } + return env +} + +func getCategoryForAStruct(genDecl *ast.GenDecl) string { + category := defaultCategory + if genDecl.Doc != nil { + commentTexts := strings.Split(genDecl.Doc.Text(), "\n") + for _, comment := range commentTexts { + commentText := strings.TrimPrefix(strings.ReplaceAll(comment, " ", ""), "//") // this can happen if comment group is in /* */ + if strings.HasPrefix(commentText, categoryCommentStructPrefix) { + categories := strings.Split(strings.TrimPrefix(commentText, categoryCommentStructPrefix), ",") + if len(categories) > 0 && len(categories[0]) > 0 { //only supporting one category as of now + category = categories[0] //overriding category + break + } + } + } + } + return category +} + +func addReadmeTableDelimiterEscapeChar(s string) string { + return strings.ReplaceAll(s, "|", `\|`) +} + +func writeToFile(categoryFieldsMap map[string][]EnvField) { + cfs := make([]CategoryField, 0, len(categoryFieldsMap)) + for category, allFields := range categoryFieldsMap { + sort.Slice(allFields, func(i, j int) bool { + return allFields[i].Env < allFields[j].Env + }) + + cfs = append(cfs, CategoryField{ + Category: category, + Fields: allFields, + }) + } + sort.Slice(cfs, func(i, j int) bool { + return cfs[i].Category < cfs[j].Category + }) + file, err := os.Create(MARKDOWN_FILENAME) + if err != nil && !errors.Is(err, os.ErrExist) { + panic(err) + } + defer file.Close() + tmpl, err := template.New("markdown").Parse(MarkdownTemplate) + if err != nil { + panic(err) + } + err = tmpl.Execute(file, cfs) + if err != nil { + panic(err) + } + cfsMarshaled, err := json.Marshal(cfs) + if err != nil { + log.Println("error marshalling category fields:", err) + panic(err) + } + err = os.WriteFile(MARKDOWN_JSON_FILENAME, cfsMarshaled, 0644) + if err != nil { + panic(err) + } +} diff --git a/kubelink/vendor/modules.txt b/kubelink/vendor/modules.txt index ceefba27e..221c8b7d4 100644 --- a/kubelink/vendor/modules.txt +++ b/kubelink/vendor/modules.txt @@ -127,10 +127,11 @@ github.com/cyphar/filepath-securejoin # github.com/davecgh/go-spew v1.1.1 ## explicit github.com/davecgh/go-spew/spew -# github.com/devtron-labs/common-lib v0.0.0 => github.com/devtron-labs/devtron-services/common-lib v0.0.0-20250127074233-945aa5b576dd +# github.com/devtron-labs/common-lib v0.0.0 => github.com/devtron-labs/devtron-services/common-lib v0.0.0-20250127103321-97c55f5baa57 ## explicit; go 1.21 github.com/devtron-labs/common-lib/async github.com/devtron-labs/common-lib/constants +github.com/devtron-labs/common-lib/fetchAllEnv github.com/devtron-labs/common-lib/git-manager/util github.com/devtron-labs/common-lib/helmLib/registry github.com/devtron-labs/common-lib/middlewares @@ -1353,7 +1354,7 @@ sigs.k8s.io/structured-merge-diff/v4/value # sigs.k8s.io/yaml v1.3.0 ## explicit; go 1.12 sigs.k8s.io/yaml -# github.com/devtron-labs/common-lib => github.com/devtron-labs/devtron-services/common-lib v0.0.0-20250127074233-945aa5b576dd +# github.com/devtron-labs/common-lib => github.com/devtron-labs/devtron-services/common-lib v0.0.0-20250127103321-97c55f5baa57 # go.opentelemetry.io/otel/metric => go.opentelemetry.io/otel/metric v1.18.0 # k8s.io/api => k8s.io/api v0.29.0 # k8s.io/apiextensions-apiserver => k8s.io/apiextensions-apiserver v0.29.0 diff --git a/kubelink/wire_gen.go b/kubelink/wire_gen.go index c8075adb9..2ded63c44 100644 --- a/kubelink/wire_gen.go +++ b/kubelink/wire_gen.go @@ -1,6 +1,6 @@ // Code generated by Wire. DO NOT EDIT. -//go:generate go run github.com/google/wire/cmd/wire +//go:generate go run -mod=mod github.com/google/wire/cmd/wire //go:build !wireinject // +build !wireinject diff --git a/kubewatch/Makefile b/kubewatch/Makefile index 34d7725b4..5d74a28bb 100644 --- a/kubewatch/Makefile +++ b/kubewatch/Makefile @@ -13,7 +13,7 @@ GOFLAGS ?= $(GOFLAGS:) LDFLAGS := "-X '$(PKG)/cmd.gitCommit=$(TRAVIS_COMMIT)' \ -X '$(PKG)/cmd.buildDate=$(BUILD_DATE)'" -default: build +default: fetch-all-env build build: CGO_ENABLED=0 GOOS=linux "$(GOCMD)" build ${GOFLAGS} -ldflags ${LDFLAGS} -o "${BINARY}" @@ -36,3 +36,6 @@ dep-update-oss: go mod edit -replace=github.com/devtron-labs/common-lib=github.com/devtron-labs/devtron-services/common-lib@$(TARGET_BRANCH) go mod tidy go mod vendor + +fetch-all-env: + go run fetchAllEnv/fetchAllEnv.go diff --git a/kubewatch/env_gen.json b/kubewatch/env_gen.json new file mode 100644 index 000000000..937a9036f --- /dev/null +++ b/kubewatch/env_gen.json @@ -0,0 +1 @@ +[{"Category":"DEVTRON","Fields":[{"Env":"ACD_INFORMER","EnvType":"bool","EnvValue":"true","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"ACD_NAMESPACE","EnvType":"string","EnvValue":"devtroncd","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"APP","EnvType":"string","EnvValue":"kubewatch","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"CD_DEFAULT_NAMESPACE","EnvType":"string","EnvValue":"devtron-cd","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"CD_EXTERNAL_LISTENER_URL","EnvType":"string","EnvValue":"http://devtroncd-orchestrator-service-prod.devtroncd:80","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"CD_EXTERNAL_NAMESPACE","EnvType":"string","EnvValue":"","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"CD_EXTERNAL_ORCHESTRATOR_TOKEN","EnvType":"string","EnvValue":"","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"CD_EXTERNAL_REST_LISTENER","EnvType":"bool","EnvValue":"false","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"CD_INFORMER","EnvType":"bool","EnvValue":"true","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"CI_INFORMER","EnvType":"bool","EnvValue":"true","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"CLUSTER_TYPE","EnvType":"string","EnvValue":"IN_CLUSTER","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"CONSUMER_CONFIG_JSON","EnvType":"string","EnvValue":"","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"DEFAULT_LOG_TIME_LIMIT","EnvType":"int64","EnvValue":"1","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"DEFAULT_NAMESPACE","EnvType":"string","EnvValue":"devtron-ci","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"ENABLE_STATSVIZ","EnvType":"bool","EnvValue":"false","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"K8s_CLIENT_MAX_IDLE_CONNS_PER_HOST","EnvType":"int","EnvValue":"25","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"K8s_TCP_IDLE_CONN_TIMEOUT","EnvType":"int","EnvValue":"300","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"K8s_TCP_KEEPALIVE","EnvType":"int","EnvValue":"30","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"K8s_TCP_TIMEOUT","EnvType":"int","EnvValue":"30","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"K8s_TLS_HANDSHAKE_TIMEOUT","EnvType":"int","EnvValue":"10","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"LOG_LEVEL","EnvType":"int","EnvValue":"-1","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"NATS_MSG_ACK_WAIT_IN_SECS","EnvType":"int","EnvValue":"120","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"NATS_MSG_BUFFER_SIZE","EnvType":"int","EnvValue":"-1","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"NATS_MSG_MAX_AGE","EnvType":"int","EnvValue":"86400","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"NATS_MSG_PROCESSING_BATCH_SIZE","EnvType":"int","EnvValue":"1","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"NATS_MSG_REPLICAS","EnvType":"int","EnvValue":"0","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"NATS_SERVER_HOST","EnvType":"string","EnvValue":"nats://devtron-nats.devtroncd:4222","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"PG_ADDR","EnvType":"string","EnvValue":"127.0.0.1","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"PG_DATABASE","EnvType":"string","EnvValue":"orchestrator","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"PG_EXPORT_PROM_METRICS","EnvType":"bool","EnvValue":"true","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"PG_LOG_ALL_FAILURE_QUERIES","EnvType":"bool","EnvValue":"true","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"PG_LOG_ALL_QUERY","EnvType":"bool","EnvValue":"false","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"PG_LOG_SLOW_QUERY","EnvType":"bool","EnvValue":"true","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"PG_PASSWORD","EnvType":"string","EnvValue":"","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"PG_PORT","EnvType":"string","EnvValue":"5432","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"PG_QUERY_DUR_THRESHOLD","EnvType":"int64","EnvValue":"5000","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"PG_USER","EnvType":"string","EnvValue":"","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"RUNTIME_CONFIG_LOCAL_DEV","EnvType":"LocalDevMode","EnvValue":"false","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"SLEEP_TIMEOUT","EnvType":"int","EnvValue":"5","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"STREAM_CONFIG_JSON","EnvType":"string","EnvValue":"","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"USE_CUSTOM_HTTP_TRANSPORT","EnvType":"bool","EnvValue":"false","EnvDescription":"","Example":"","Deprecated":"false"}]}] \ No newline at end of file diff --git a/kubewatch/env_gen.md b/kubewatch/env_gen.md new file mode 100644 index 000000000..6fa67c8b0 --- /dev/null +++ b/kubewatch/env_gen.md @@ -0,0 +1,47 @@ + + +## DEVTRON Related Environment Variables +| Key | Type | Default Value | Description | Example | Deprecated | +|-------|----------|-------------------|-------------------|-----------------------|------------------| + | ACD_INFORMER | bool |true | | | false | + | ACD_NAMESPACE | string |devtroncd | | | false | + | APP | string |kubewatch | | | false | + | CD_DEFAULT_NAMESPACE | string |devtron-cd | | | false | + | CD_EXTERNAL_LISTENER_URL | string |http://devtroncd-orchestrator-service-prod.devtroncd:80 | | | false | + | CD_EXTERNAL_NAMESPACE | string | | | | false | + | CD_EXTERNAL_ORCHESTRATOR_TOKEN | string | | | | false | + | CD_EXTERNAL_REST_LISTENER | bool |false | | | false | + | CD_INFORMER | bool |true | | | false | + | CI_INFORMER | bool |true | | | false | + | CLUSTER_TYPE | string |IN_CLUSTER | | | false | + | CONSUMER_CONFIG_JSON | string | | | | false | + | DEFAULT_LOG_TIME_LIMIT | int64 |1 | | | false | + | DEFAULT_NAMESPACE | string |devtron-ci | | | false | + | ENABLE_STATSVIZ | bool |false | | | false | + | K8s_CLIENT_MAX_IDLE_CONNS_PER_HOST | int |25 | | | false | + | K8s_TCP_IDLE_CONN_TIMEOUT | int |300 | | | false | + | K8s_TCP_KEEPALIVE | int |30 | | | false | + | K8s_TCP_TIMEOUT | int |30 | | | false | + | K8s_TLS_HANDSHAKE_TIMEOUT | int |10 | | | false | + | LOG_LEVEL | int |-1 | | | false | + | NATS_MSG_ACK_WAIT_IN_SECS | int |120 | | | false | + | NATS_MSG_BUFFER_SIZE | int |-1 | | | false | + | NATS_MSG_MAX_AGE | int |86400 | | | false | + | NATS_MSG_PROCESSING_BATCH_SIZE | int |1 | | | false | + | NATS_MSG_REPLICAS | int |0 | | | false | + | NATS_SERVER_HOST | string |nats://devtron-nats.devtroncd:4222 | | | false | + | PG_ADDR | string |127.0.0.1 | | | false | + | PG_DATABASE | string |orchestrator | | | false | + | PG_EXPORT_PROM_METRICS | bool |true | | | false | + | PG_LOG_ALL_FAILURE_QUERIES | bool |true | | | false | + | PG_LOG_ALL_QUERY | bool |false | | | false | + | PG_LOG_SLOW_QUERY | bool |true | | | false | + | PG_PASSWORD | string | | | | false | + | PG_PORT | string |5432 | | | false | + | PG_QUERY_DUR_THRESHOLD | int64 |5000 | | | false | + | PG_USER | string | | | | false | + | RUNTIME_CONFIG_LOCAL_DEV | LocalDevMode |false | | | false | + | SLEEP_TIMEOUT | int |5 | | | false | + | STREAM_CONFIG_JSON | string | | | | false | + | USE_CUSTOM_HTTP_TRANSPORT | bool |false | | | false | + diff --git a/kubewatch/fetchAllEnv/fetchAllEnv.go b/kubewatch/fetchAllEnv/fetchAllEnv.go new file mode 100644 index 000000000..ed6163dc6 --- /dev/null +++ b/kubewatch/fetchAllEnv/fetchAllEnv.go @@ -0,0 +1,25 @@ +/* + * Copyright (c) 2024. Devtron Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package main + +import ( + "github.com/devtron-labs/common-lib/fetchAllEnv" +) + +func main() { + fetchAllEnv.FetchEnvAndWriteToFile() +} diff --git a/kubewatch/go.mod b/kubewatch/go.mod index 5699ae84e..ca5e4579b 100644 --- a/kubewatch/go.mod +++ b/kubewatch/go.mod @@ -245,4 +245,4 @@ replace ( k8s.io/sample-apiserver => k8s.io/sample-apiserver v0.29.7 ) -replace github.com/devtron-labs/common-lib => github.com/devtron-labs/devtron-services/common-lib v0.0.0-20250127074233-945aa5b576dd +replace github.com/devtron-labs/common-lib => github.com/devtron-labs/devtron-services/common-lib v0.0.0-20250127103321-97c55f5baa57 diff --git a/kubewatch/go.sum b/kubewatch/go.sum index a45e18ad3..0436f6be8 100644 --- a/kubewatch/go.sum +++ b/kubewatch/go.sum @@ -719,8 +719,8 @@ github.com/cyphar/filepath-securejoin v0.2.4/go.mod h1:aPGpWjXOXUn2NCNjFvBE6aRxG github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= -github.com/devtron-labs/devtron-services/common-lib v0.0.0-20250127074233-945aa5b576dd h1:Mq9sBScfBWU4FWnfihutOXQbvy3t63tViYN9DVrPExw= -github.com/devtron-labs/devtron-services/common-lib v0.0.0-20250127074233-945aa5b576dd/go.mod h1:1QJJLpgJSkb5Jm9xPeKAk+kXb0QgBOOOgJj0cgYhAVA= +github.com/devtron-labs/devtron-services/common-lib v0.0.0-20250127103321-97c55f5baa57 h1:EPMILz/Lk4oK/gTJWiOL5a7jFAiuiokeFFc/T7qeQ/o= +github.com/devtron-labs/devtron-services/common-lib v0.0.0-20250127103321-97c55f5baa57/go.mod h1:1QJJLpgJSkb5Jm9xPeKAk+kXb0QgBOOOgJj0cgYhAVA= github.com/dgryski/go-rendezvous v0.0.0-20200823014737-9f7001d12a5f h1:lO4WD4F/rVNCu3HqELle0jiPLLBs70cWOduZpkS1E78= github.com/dgryski/go-rendezvous v0.0.0-20200823014737-9f7001d12a5f/go.mod h1:cuUVRXasLTGF7a8hSLbxyZXjz+1KgoB3wDUb6vlszIc= github.com/distribution/reference v0.6.0 h1:0IXCQ5g4/QMHHkarYzh5l+u8T3t73zM5QvfrDyIgxBk= diff --git a/kubewatch/vendor/github.com/devtron-labs/common-lib/fetchAllEnv/fetchAllEnv.go b/kubewatch/vendor/github.com/devtron-labs/common-lib/fetchAllEnv/fetchAllEnv.go new file mode 100644 index 000000000..c8a19e282 --- /dev/null +++ b/kubewatch/vendor/github.com/devtron-labs/common-lib/fetchAllEnv/fetchAllEnv.go @@ -0,0 +1,226 @@ +/* + * Copyright (c) 2024. Devtron Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package fetchAllEnv + +import ( + "encoding/json" + "errors" + "go/ast" + "go/parser" + "go/token" + "log" + "os" + "path/filepath" + "reflect" + "sort" + "strings" + "text/template" +) + +type EnvField struct { + Env string + EnvType string + EnvValue string + EnvDescription string + Example string + Deprecated string +} + +type CategoryField struct { + Category string + Fields []EnvField +} + +const ( + categoryCommentStructPrefix = "CATEGORY=" + defaultCategory = "DEVTRON" + deprecatedDefaultValue = "false" + + envFieldTypeTag = "env" + envDefaultFieldTypeTag = "envDefault" + envDescriptionFieldTypeTag = "description" + envPossibleValuesFieldTypeTag = "example" + envDeprecatedFieldTypeTag = "deprecated" + MARKDOWN_FILENAME = "env_gen.md" + MARKDOWN_JSON_FILENAME = "env_gen.json" +) + +const MarkdownTemplate = ` +{{range . }} +## {{ .Category }} Related Environment Variables +| Key | Type | Default Value | Description | Example | Deprecated | +|-------|----------|-------------------|-------------------|-----------------------|------------------| +{{range .Fields }} | {{ .Env }} | {{ .EnvType }} |{{ .EnvValue }} | {{ .EnvDescription }} | {{ .Example }} | {{ .Deprecated }} | +{{end}} +{{end}}` + +func FetchEnvAndWriteToFile() { + WalkThroughProject() + return +} + +func WalkThroughProject() { + categoryFieldsMap := make(map[string][]EnvField) + uniqueKeys := make(map[string]bool) + err := filepath.Walk(".", func(path string, info os.FileInfo, err error) error { + if err != nil { + return err + } + if !info.IsDir() && strings.HasSuffix(path, ".go") { + err = processGoFile(path, categoryFieldsMap, uniqueKeys) + if err != nil { + log.Println("error in processing go file", err) + return err + } + } + return nil + }) + if err != nil { + return + } + writeToFile(categoryFieldsMap) +} + +func processGoFile(filePath string, categoryFieldsMap map[string][]EnvField, uniqueKeys map[string]bool) error { + fset := token.NewFileSet() + node, err := parser.ParseFile(fset, filePath, nil, parser.ParseComments) + if err != nil { + log.Println("error parsing file:", err) + return err + } + ast.Inspect(node, func(n ast.Node) bool { + if genDecl, ok := n.(*ast.GenDecl); ok { + // checking if type declaration, one of [func, map, struct, array, channel, interface] + if genDecl.Tok == token.TYPE { + for _, spec := range genDecl.Specs { + if typeSpec, ok := spec.(*ast.TypeSpec); ok { + // only checking struct type declarations + if structType, ok2 := typeSpec.Type.(*ast.StructType); ok2 { + allFields := make([]EnvField, 0, len(structType.Fields.List)) + for _, field := range structType.Fields.List { + if field.Tag != nil { + envField := getEnvKeyAndValue(field) + envKey := envField.Env + if len(envKey) == 0 || uniqueKeys[envKey] { + continue + } + allFields = append(allFields, envField) + uniqueKeys[envKey] = true + } + } + if len(allFields) > 0 { + category := getCategoryForAStruct(genDecl) + categoryFieldsMap[category] = append(categoryFieldsMap[category], allFields...) + } + } + } + } + } + } + return true + }) + return nil +} + +func getEnvKeyAndValue(field *ast.Field) EnvField { + tag := reflect.StructTag(strings.Trim(field.Tag.Value, "`")) // remove surrounding backticks + + envKey := addReadmeTableDelimiterEscapeChar(tag.Get(envFieldTypeTag)) + envValue := addReadmeTableDelimiterEscapeChar(tag.Get(envDefaultFieldTypeTag)) + envDescription := addReadmeTableDelimiterEscapeChar(tag.Get(envDescriptionFieldTypeTag)) + envPossibleValues := addReadmeTableDelimiterEscapeChar(tag.Get(envPossibleValuesFieldTypeTag)) + envDeprecated := addReadmeTableDelimiterEscapeChar(tag.Get(envDeprecatedFieldTypeTag)) + // check if there exist any value provided in env for this field + if value, ok := os.LookupEnv(envKey); ok { + envValue = value + } + env := EnvField{ + Env: envKey, + EnvValue: envValue, + EnvDescription: envDescription, + Example: envPossibleValues, + Deprecated: envDeprecated, + } + if indent, ok := field.Type.(*ast.Ident); ok && indent != nil { + env.EnvType = indent.Name + } + if len(envDeprecated) == 0 { + env.Deprecated = deprecatedDefaultValue + } + return env +} + +func getCategoryForAStruct(genDecl *ast.GenDecl) string { + category := defaultCategory + if genDecl.Doc != nil { + commentTexts := strings.Split(genDecl.Doc.Text(), "\n") + for _, comment := range commentTexts { + commentText := strings.TrimPrefix(strings.ReplaceAll(comment, " ", ""), "//") // this can happen if comment group is in /* */ + if strings.HasPrefix(commentText, categoryCommentStructPrefix) { + categories := strings.Split(strings.TrimPrefix(commentText, categoryCommentStructPrefix), ",") + if len(categories) > 0 && len(categories[0]) > 0 { //only supporting one category as of now + category = categories[0] //overriding category + break + } + } + } + } + return category +} + +func addReadmeTableDelimiterEscapeChar(s string) string { + return strings.ReplaceAll(s, "|", `\|`) +} + +func writeToFile(categoryFieldsMap map[string][]EnvField) { + cfs := make([]CategoryField, 0, len(categoryFieldsMap)) + for category, allFields := range categoryFieldsMap { + sort.Slice(allFields, func(i, j int) bool { + return allFields[i].Env < allFields[j].Env + }) + + cfs = append(cfs, CategoryField{ + Category: category, + Fields: allFields, + }) + } + sort.Slice(cfs, func(i, j int) bool { + return cfs[i].Category < cfs[j].Category + }) + file, err := os.Create(MARKDOWN_FILENAME) + if err != nil && !errors.Is(err, os.ErrExist) { + panic(err) + } + defer file.Close() + tmpl, err := template.New("markdown").Parse(MarkdownTemplate) + if err != nil { + panic(err) + } + err = tmpl.Execute(file, cfs) + if err != nil { + panic(err) + } + cfsMarshaled, err := json.Marshal(cfs) + if err != nil { + log.Println("error marshalling category fields:", err) + panic(err) + } + err = os.WriteFile(MARKDOWN_JSON_FILENAME, cfsMarshaled, 0644) + if err != nil { + panic(err) + } +} diff --git a/kubewatch/vendor/modules.txt b/kubewatch/vendor/modules.txt index e3ecf42c7..3682c833c 100644 --- a/kubewatch/vendor/modules.txt +++ b/kubewatch/vendor/modules.txt @@ -211,9 +211,10 @@ github.com/cyphar/filepath-securejoin # github.com/davecgh/go-spew v1.1.1 ## explicit github.com/davecgh/go-spew/spew -# github.com/devtron-labs/common-lib v0.0.0 => github.com/devtron-labs/devtron-services/common-lib v0.0.0-20250127074233-945aa5b576dd +# github.com/devtron-labs/common-lib v0.0.0 => github.com/devtron-labs/devtron-services/common-lib v0.0.0-20250127103321-97c55f5baa57 ## explicit; go 1.21 github.com/devtron-labs/common-lib/constants +github.com/devtron-labs/common-lib/fetchAllEnv github.com/devtron-labs/common-lib/git-manager/util github.com/devtron-labs/common-lib/monitoring github.com/devtron-labs/common-lib/monitoring/pprof @@ -1753,4 +1754,4 @@ upper.io/db.v3/postgresql # k8s.io/mount-utils => k8s.io/mount-utils v0.29.7 # k8s.io/pod-security-admission => k8s.io/pod-security-admission v0.29.7 # k8s.io/sample-apiserver => k8s.io/sample-apiserver v0.29.7 -# github.com/devtron-labs/common-lib => github.com/devtron-labs/devtron-services/common-lib v0.0.0-20250127074233-945aa5b576dd +# github.com/devtron-labs/common-lib => github.com/devtron-labs/devtron-services/common-lib v0.0.0-20250127103321-97c55f5baa57 diff --git a/lens/Makefile b/lens/Makefile index 65dfa1bae..498f98d70 100644 --- a/lens/Makefile +++ b/lens/Makefile @@ -1,5 +1,5 @@ -all: build +all: fetch-all-env build TAG?=latest FLAGS= @@ -34,4 +34,8 @@ TARGET_BRANCH?=main dep-update-oss: go mod edit -replace=github.com/devtron-labs/common-lib=github.com/devtron-labs/devtron-services/common-lib@$(TARGET_BRANCH) go mod tidy - go mod vendor \ No newline at end of file + go mod vendor + + +fetch-all-env: + go run fetchAllEnv/fetchAllEnv.go diff --git a/lens/env_gen.json b/lens/env_gen.json new file mode 100644 index 000000000..e226c0dad --- /dev/null +++ b/lens/env_gen.json @@ -0,0 +1 @@ +[{"Category":"DEVTRON","Fields":[{"Env":"APP","EnvType":"string","EnvValue":"lens","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"CONSUMER_CONFIG_JSON","EnvType":"string","EnvValue":"","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"DEFAULT_LOG_TIME_LIMIT","EnvType":"int64","EnvValue":"1","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"GIT_SENSOR_PROTOCOL","EnvType":"string","EnvValue":"REST","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"GIT_SENSOR_TIMEOUT","EnvType":"int","EnvValue":"0","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"GIT_SENSOR_URL","EnvType":"string","EnvValue":"http://localhost:9999","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"LOG_LEVEL","EnvType":"int","EnvValue":"0","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"NATS_MSG_ACK_WAIT_IN_SECS","EnvType":"int","EnvValue":"120","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"NATS_MSG_BUFFER_SIZE","EnvType":"int","EnvValue":"-1","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"NATS_MSG_MAX_AGE","EnvType":"int","EnvValue":"86400","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"NATS_MSG_PROCESSING_BATCH_SIZE","EnvType":"int","EnvValue":"1","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"NATS_MSG_REPLICAS","EnvType":"int","EnvValue":"0","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"NATS_SERVER_HOST","EnvType":"string","EnvValue":"nats://devtron-nats.devtroncd:4222","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"PG_ADDR","EnvType":"string","EnvValue":"127.0.0.1","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"PG_DATABASE","EnvType":"string","EnvValue":"lens","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"PG_EXPORT_PROM_METRICS","EnvType":"bool","EnvValue":"true","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"PG_LOG_ALL_FAILURE_QUERIES","EnvType":"bool","EnvValue":"true","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"PG_LOG_ALL_QUERY","EnvType":"bool","EnvValue":"false","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"PG_LOG_SLOW_QUERY","EnvType":"bool","EnvValue":"true","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"PG_PASSWORD","EnvType":"string","EnvValue":"","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"PG_PORT","EnvType":"string","EnvValue":"5432","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"PG_QUERY_DUR_THRESHOLD","EnvType":"int64","EnvValue":"5000","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"PG_USER","EnvType":"string","EnvValue":"","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"STREAM_CONFIG_JSON","EnvType":"string","EnvValue":"","EnvDescription":"","Example":"","Deprecated":"false"}]}] \ No newline at end of file diff --git a/lens/env_gen.md b/lens/env_gen.md new file mode 100644 index 000000000..1f414f667 --- /dev/null +++ b/lens/env_gen.md @@ -0,0 +1,30 @@ + + +## DEVTRON Related Environment Variables +| Key | Type | Default Value | Description | Example | Deprecated | +|-------|----------|-------------------|-------------------|-----------------------|------------------| + | APP | string |lens | | | false | + | CONSUMER_CONFIG_JSON | string | | | | false | + | DEFAULT_LOG_TIME_LIMIT | int64 |1 | | | false | + | GIT_SENSOR_PROTOCOL | string |REST | | | false | + | GIT_SENSOR_TIMEOUT | int |0 | | | false | + | GIT_SENSOR_URL | string |http://localhost:9999 | | | false | + | LOG_LEVEL | int |0 | | | false | + | NATS_MSG_ACK_WAIT_IN_SECS | int |120 | | | false | + | NATS_MSG_BUFFER_SIZE | int |-1 | | | false | + | NATS_MSG_MAX_AGE | int |86400 | | | false | + | NATS_MSG_PROCESSING_BATCH_SIZE | int |1 | | | false | + | NATS_MSG_REPLICAS | int |0 | | | false | + | NATS_SERVER_HOST | string |nats://devtron-nats.devtroncd:4222 | | | false | + | PG_ADDR | string |127.0.0.1 | | | false | + | PG_DATABASE | string |lens | | | false | + | PG_EXPORT_PROM_METRICS | bool |true | | | false | + | PG_LOG_ALL_FAILURE_QUERIES | bool |true | | | false | + | PG_LOG_ALL_QUERY | bool |false | | | false | + | PG_LOG_SLOW_QUERY | bool |true | | | false | + | PG_PASSWORD | string | | | | false | + | PG_PORT | string |5432 | | | false | + | PG_QUERY_DUR_THRESHOLD | int64 |5000 | | | false | + | PG_USER | string | | | | false | + | STREAM_CONFIG_JSON | string | | | | false | + diff --git a/lens/fetchAllEnv/fetchAllEnv.go b/lens/fetchAllEnv/fetchAllEnv.go new file mode 100644 index 000000000..ed6163dc6 --- /dev/null +++ b/lens/fetchAllEnv/fetchAllEnv.go @@ -0,0 +1,25 @@ +/* + * Copyright (c) 2024. Devtron Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package main + +import ( + "github.com/devtron-labs/common-lib/fetchAllEnv" +) + +func main() { + fetchAllEnv.FetchEnvAndWriteToFile() +} diff --git a/lens/go.mod b/lens/go.mod index d75e7e164..2425b4537 100644 --- a/lens/go.mod +++ b/lens/go.mod @@ -59,4 +59,4 @@ require ( github.com/onsi/gomega v1.18.1 // indirect ) -replace github.com/devtron-labs/common-lib => github.com/devtron-labs/devtron-services/common-lib v0.0.0-20250127074233-945aa5b576dd +replace github.com/devtron-labs/common-lib => github.com/devtron-labs/devtron-services/common-lib v0.0.0-20250127103321-97c55f5baa57 diff --git a/lens/go.sum b/lens/go.sum index 666f294ed..b3db4994c 100644 --- a/lens/go.sum +++ b/lens/go.sum @@ -22,8 +22,8 @@ github.com/cncf/xds/go v0.0.0-20230607035331-e9ce68804cb4/go.mod h1:eXthEFrGJvWH github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= -github.com/devtron-labs/devtron-services/common-lib v0.0.0-20250127074233-945aa5b576dd h1:Mq9sBScfBWU4FWnfihutOXQbvy3t63tViYN9DVrPExw= -github.com/devtron-labs/devtron-services/common-lib v0.0.0-20250127074233-945aa5b576dd/go.mod h1:1QJJLpgJSkb5Jm9xPeKAk+kXb0QgBOOOgJj0cgYhAVA= +github.com/devtron-labs/devtron-services/common-lib v0.0.0-20250127103321-97c55f5baa57 h1:EPMILz/Lk4oK/gTJWiOL5a7jFAiuiokeFFc/T7qeQ/o= +github.com/devtron-labs/devtron-services/common-lib v0.0.0-20250127103321-97c55f5baa57/go.mod h1:1QJJLpgJSkb5Jm9xPeKAk+kXb0QgBOOOgJj0cgYhAVA= github.com/devtron-labs/protos v0.0.3-0.20240130061723-7b2e12ab0abb h1:CkfQQgZc950/hTPqtQSiHV2RmZgkBLGCzwR02FZYjAU= github.com/devtron-labs/protos v0.0.3-0.20240130061723-7b2e12ab0abb/go.mod h1:pjLjgoa1GzbkOkvbMyP4SAKsaiK7eG6GoQCNauG03JA= github.com/docker/cli v24.0.6+incompatible h1:fF+XCQCgJjjQNIMjzaSmiKJSCcfcXb3TWTcc7GAneOY= diff --git a/lens/vendor/github.com/devtron-labs/common-lib/fetchAllEnv/fetchAllEnv.go b/lens/vendor/github.com/devtron-labs/common-lib/fetchAllEnv/fetchAllEnv.go new file mode 100644 index 000000000..c8a19e282 --- /dev/null +++ b/lens/vendor/github.com/devtron-labs/common-lib/fetchAllEnv/fetchAllEnv.go @@ -0,0 +1,226 @@ +/* + * Copyright (c) 2024. Devtron Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package fetchAllEnv + +import ( + "encoding/json" + "errors" + "go/ast" + "go/parser" + "go/token" + "log" + "os" + "path/filepath" + "reflect" + "sort" + "strings" + "text/template" +) + +type EnvField struct { + Env string + EnvType string + EnvValue string + EnvDescription string + Example string + Deprecated string +} + +type CategoryField struct { + Category string + Fields []EnvField +} + +const ( + categoryCommentStructPrefix = "CATEGORY=" + defaultCategory = "DEVTRON" + deprecatedDefaultValue = "false" + + envFieldTypeTag = "env" + envDefaultFieldTypeTag = "envDefault" + envDescriptionFieldTypeTag = "description" + envPossibleValuesFieldTypeTag = "example" + envDeprecatedFieldTypeTag = "deprecated" + MARKDOWN_FILENAME = "env_gen.md" + MARKDOWN_JSON_FILENAME = "env_gen.json" +) + +const MarkdownTemplate = ` +{{range . }} +## {{ .Category }} Related Environment Variables +| Key | Type | Default Value | Description | Example | Deprecated | +|-------|----------|-------------------|-------------------|-----------------------|------------------| +{{range .Fields }} | {{ .Env }} | {{ .EnvType }} |{{ .EnvValue }} | {{ .EnvDescription }} | {{ .Example }} | {{ .Deprecated }} | +{{end}} +{{end}}` + +func FetchEnvAndWriteToFile() { + WalkThroughProject() + return +} + +func WalkThroughProject() { + categoryFieldsMap := make(map[string][]EnvField) + uniqueKeys := make(map[string]bool) + err := filepath.Walk(".", func(path string, info os.FileInfo, err error) error { + if err != nil { + return err + } + if !info.IsDir() && strings.HasSuffix(path, ".go") { + err = processGoFile(path, categoryFieldsMap, uniqueKeys) + if err != nil { + log.Println("error in processing go file", err) + return err + } + } + return nil + }) + if err != nil { + return + } + writeToFile(categoryFieldsMap) +} + +func processGoFile(filePath string, categoryFieldsMap map[string][]EnvField, uniqueKeys map[string]bool) error { + fset := token.NewFileSet() + node, err := parser.ParseFile(fset, filePath, nil, parser.ParseComments) + if err != nil { + log.Println("error parsing file:", err) + return err + } + ast.Inspect(node, func(n ast.Node) bool { + if genDecl, ok := n.(*ast.GenDecl); ok { + // checking if type declaration, one of [func, map, struct, array, channel, interface] + if genDecl.Tok == token.TYPE { + for _, spec := range genDecl.Specs { + if typeSpec, ok := spec.(*ast.TypeSpec); ok { + // only checking struct type declarations + if structType, ok2 := typeSpec.Type.(*ast.StructType); ok2 { + allFields := make([]EnvField, 0, len(structType.Fields.List)) + for _, field := range structType.Fields.List { + if field.Tag != nil { + envField := getEnvKeyAndValue(field) + envKey := envField.Env + if len(envKey) == 0 || uniqueKeys[envKey] { + continue + } + allFields = append(allFields, envField) + uniqueKeys[envKey] = true + } + } + if len(allFields) > 0 { + category := getCategoryForAStruct(genDecl) + categoryFieldsMap[category] = append(categoryFieldsMap[category], allFields...) + } + } + } + } + } + } + return true + }) + return nil +} + +func getEnvKeyAndValue(field *ast.Field) EnvField { + tag := reflect.StructTag(strings.Trim(field.Tag.Value, "`")) // remove surrounding backticks + + envKey := addReadmeTableDelimiterEscapeChar(tag.Get(envFieldTypeTag)) + envValue := addReadmeTableDelimiterEscapeChar(tag.Get(envDefaultFieldTypeTag)) + envDescription := addReadmeTableDelimiterEscapeChar(tag.Get(envDescriptionFieldTypeTag)) + envPossibleValues := addReadmeTableDelimiterEscapeChar(tag.Get(envPossibleValuesFieldTypeTag)) + envDeprecated := addReadmeTableDelimiterEscapeChar(tag.Get(envDeprecatedFieldTypeTag)) + // check if there exist any value provided in env for this field + if value, ok := os.LookupEnv(envKey); ok { + envValue = value + } + env := EnvField{ + Env: envKey, + EnvValue: envValue, + EnvDescription: envDescription, + Example: envPossibleValues, + Deprecated: envDeprecated, + } + if indent, ok := field.Type.(*ast.Ident); ok && indent != nil { + env.EnvType = indent.Name + } + if len(envDeprecated) == 0 { + env.Deprecated = deprecatedDefaultValue + } + return env +} + +func getCategoryForAStruct(genDecl *ast.GenDecl) string { + category := defaultCategory + if genDecl.Doc != nil { + commentTexts := strings.Split(genDecl.Doc.Text(), "\n") + for _, comment := range commentTexts { + commentText := strings.TrimPrefix(strings.ReplaceAll(comment, " ", ""), "//") // this can happen if comment group is in /* */ + if strings.HasPrefix(commentText, categoryCommentStructPrefix) { + categories := strings.Split(strings.TrimPrefix(commentText, categoryCommentStructPrefix), ",") + if len(categories) > 0 && len(categories[0]) > 0 { //only supporting one category as of now + category = categories[0] //overriding category + break + } + } + } + } + return category +} + +func addReadmeTableDelimiterEscapeChar(s string) string { + return strings.ReplaceAll(s, "|", `\|`) +} + +func writeToFile(categoryFieldsMap map[string][]EnvField) { + cfs := make([]CategoryField, 0, len(categoryFieldsMap)) + for category, allFields := range categoryFieldsMap { + sort.Slice(allFields, func(i, j int) bool { + return allFields[i].Env < allFields[j].Env + }) + + cfs = append(cfs, CategoryField{ + Category: category, + Fields: allFields, + }) + } + sort.Slice(cfs, func(i, j int) bool { + return cfs[i].Category < cfs[j].Category + }) + file, err := os.Create(MARKDOWN_FILENAME) + if err != nil && !errors.Is(err, os.ErrExist) { + panic(err) + } + defer file.Close() + tmpl, err := template.New("markdown").Parse(MarkdownTemplate) + if err != nil { + panic(err) + } + err = tmpl.Execute(file, cfs) + if err != nil { + panic(err) + } + cfsMarshaled, err := json.Marshal(cfs) + if err != nil { + log.Println("error marshalling category fields:", err) + panic(err) + } + err = os.WriteFile(MARKDOWN_JSON_FILENAME, cfsMarshaled, 0644) + if err != nil { + panic(err) + } +} diff --git a/lens/vendor/modules.txt b/lens/vendor/modules.txt index 9ebbfe386..894833194 100644 --- a/lens/vendor/modules.txt +++ b/lens/vendor/modules.txt @@ -7,9 +7,10 @@ github.com/caarlos0/env # github.com/cespare/xxhash/v2 v2.2.0 ## explicit; go 1.11 github.com/cespare/xxhash/v2 -# github.com/devtron-labs/common-lib v0.0.0 => github.com/devtron-labs/devtron-services/common-lib v0.0.0-20250127074233-945aa5b576dd +# github.com/devtron-labs/common-lib v0.0.0 => github.com/devtron-labs/devtron-services/common-lib v0.0.0-20250127103321-97c55f5baa57 ## explicit; go 1.21 github.com/devtron-labs/common-lib/constants +github.com/devtron-labs/common-lib/fetchAllEnv github.com/devtron-labs/common-lib/git-manager/util github.com/devtron-labs/common-lib/pubsub-lib github.com/devtron-labs/common-lib/pubsub-lib/metrics @@ -288,4 +289,4 @@ google.golang.org/protobuf/types/known/timestamppb # mellium.im/sasl v0.3.2 ## explicit; go 1.20 mellium.im/sasl -# github.com/devtron-labs/common-lib => github.com/devtron-labs/devtron-services/common-lib v0.0.0-20250127074233-945aa5b576dd +# github.com/devtron-labs/common-lib => github.com/devtron-labs/devtron-services/common-lib v0.0.0-20250127103321-97c55f5baa57 diff --git a/lens/wire_gen.go b/lens/wire_gen.go index a95e8302a..d1526fb45 100644 --- a/lens/wire_gen.go +++ b/lens/wire_gen.go @@ -1,6 +1,6 @@ // Code generated by Wire. DO NOT EDIT. -//go:generate go run github.com/google/wire/cmd/wire +//go:generate go run -mod=mod github.com/google/wire/cmd/wire //go:build !wireinject // +build !wireinject