From 06673be8ce38aef87e11d9b6ef82baeb35974702 Mon Sep 17 00:00:00 2001 From: Patrick Zhao Date: Tue, 24 Dec 2024 10:48:52 +0800 Subject: [PATCH] fix helm deploy job vars when only deploy image in somecase Signed-off-by: Patrick Zhao --- .../core/common/repository/models/service.go | 10 ++++++---- .../repository/models/template/product.go | 7 +++++-- .../aslan/core/common/service/kube/apply.go | 9 ++++++--- .../aslan/core/common/service/kube/helm.go | 14 +++++++++----- .../core/environment/service/environment.go | 17 ++++------------- .../aslan/core/service/service/helm.go | 7 ------- .../workflow/service/workflow/workflow_v4.go | 9 ++++++++- 7 files changed, 38 insertions(+), 35 deletions(-) diff --git a/pkg/microservice/aslan/core/common/repository/models/service.go b/pkg/microservice/aslan/core/common/repository/models/service.go index d0b4af9f83..f2af261e9a 100644 --- a/pkg/microservice/aslan/core/common/repository/models/service.go +++ b/pkg/microservice/aslan/core/common/repository/models/service.go @@ -99,7 +99,8 @@ type CreateFromChartTemplate struct { YamlData *templatemodels.CustomYaml `bson:"yaml_data,omitempty" json:"yaml_data,omitempty"` TemplateName string `bson:"template_name" json:"template_name"` ServiceName string `bson:"service_name" json:"service_name"` - Variables []*Variable `bson:"variables" json:"variables"` + // custom variables in chart template + Variables []*Variable `bson:"variables" json:"variables"` } type CreateFromChartRepo struct { @@ -196,9 +197,10 @@ type ServiceTmplRevision struct { } type HelmChart struct { - Name string `bson:"name" json:"name"` - Repo string `bson:"repo" json:"repo"` - Version string `bson:"version" json:"version"` + Name string `bson:"name" json:"name"` + Repo string `bson:"repo" json:"repo"` + Version string `bson:"version" json:"version"` + // full values yaml in service ValuesYaml string `bson:"values_yaml" json:"values_yaml"` } diff --git a/pkg/microservice/aslan/core/common/repository/models/template/product.go b/pkg/microservice/aslan/core/common/repository/models/template/product.go index 3cb4bde3dc..807625367e 100644 --- a/pkg/microservice/aslan/core/common/repository/models/template/product.go +++ b/pkg/microservice/aslan/core/common/repository/models/template/product.go @@ -121,6 +121,10 @@ func (grc *GitRepoConfig) GetNamespace() string { } type CustomYaml struct { + // helm: + // - in service: user override's values in advance setting + // - in env: user input's override yaml + // k8s: variable yaml YamlContent string `bson:"yaml_content" json:"yaml_content"` RenderVariableKVs []*commontypes.RenderVariableKV `bson:"render_variable_kvs" json:"render_variable_kvs"` Source string `bson:"source" json:"source"` @@ -139,12 +143,11 @@ type ServiceRender struct { ChartRepo string `bson:"chart_repo,omitempty" json:"chart_repo,omitempty"` ChartName string `bson:"chart_name,omitempty" json:"chart_name,omitempty"` ChartVersion string `bson:"chart_version,omitempty" json:"chart_version,omitempty"` - ValuesYaml string `bson:"values_yaml,omitempty" json:"values_yaml,omitempty"` + ValuesYaml string `bson:"values_yaml,omitempty" json:"values_yaml,omitempty"` // full helm service values yaml, only record, not actually used in calculation OverrideValues string `bson:"override_values,omitempty" json:"override_values,omitempty"` // used for helm services, json-encoded string of kv value // ---- for helm services end ---- // OverrideYaml will be used in both helm and k8s projects - // In k8s this is variable_yaml OverrideYaml *CustomYaml `bson:"override_yaml,omitempty" json:"override_yaml,omitempty"` } diff --git a/pkg/microservice/aslan/core/common/service/kube/apply.go b/pkg/microservice/aslan/core/common/service/kube/apply.go index 53577bb4dd..2223209345 100644 --- a/pkg/microservice/aslan/core/common/service/kube/apply.go +++ b/pkg/microservice/aslan/core/common/service/kube/apply.go @@ -790,6 +790,9 @@ func PrepareHelmServiceData(applyParam *ResourceApplyParam) (*commonmodels.Produ OverrideYaml: &template.CustomYaml{}, } productService.Render = targetChart + } else { + productService.GetServiceRender().ValuesYaml = svcTemplate.HelmChart.ValuesYaml + productService.GetServiceRender().ChartVersion = svcTemplate.HelmChart.Version } if applyParam.UpdateServiceRevision && productService.Revision != svcTemplate.Revision { @@ -805,18 +808,18 @@ func PrepareHelmServiceData(applyParam *ResourceApplyParam) (*commonmodels.Produ } } - replaceValuesMaps := make([]map[string]interface{}, 0) + imageValuesMaps := make([]map[string]interface{}, 0) for _, targetContainer := range productService.Containers { // prepare image replace info replaceValuesMap, err := commonutil.AssignImageData(targetContainer.Image, commonutil.GetValidMatchData(targetContainer.ImagePath)) if err != nil { return nil, nil, fmt.Errorf("failed to pase image uri %s/%s, err %s", productInfo.ProductName, applyParam.ServiceName, err.Error()) } - replaceValuesMaps = append(replaceValuesMaps, replaceValuesMap) + imageValuesMaps = append(imageValuesMaps, replaceValuesMap) } // replace image into service's values.yaml - replacedValuesYaml, err := commonutil.ReplaceImage(svcTemplate.HelmChart.ValuesYaml, replaceValuesMaps...) + replacedValuesYaml, err := commonutil.ReplaceImage(svcTemplate.HelmChart.ValuesYaml, imageValuesMaps...) if err != nil { return nil, nil, fmt.Errorf("failed to replace image uri %s/%s, err %s", productInfo.ProductName, applyParam.ServiceName, err.Error()) diff --git a/pkg/microservice/aslan/core/common/service/kube/helm.go b/pkg/microservice/aslan/core/common/service/kube/helm.go index 666d3de51b..8f8d7511c4 100644 --- a/pkg/microservice/aslan/core/common/service/kube/helm.go +++ b/pkg/microservice/aslan/core/common/service/kube/helm.go @@ -144,7 +144,11 @@ func InstallOrUpgradeHelmChartWithValues(param *ReleaseInstallParam, isRetry boo } // GeneMergedValues generate values.yaml used to install or upgrade helm chart, like param in after option -f -// If fullValues is set to true, full values yaml content will be returned, this case is used to preview values when running workflows +// productSvc: contains current images info +// svcRender: contains env values info, including service's values and env's override values +// defaultValues: global values yaml +// images: images to be replaced +// fullValues: If fullValues is set to true, full values yaml content will be returned, this case is used to preview values when running workflows func GeneMergedValues(productSvc *commonmodels.ProductService, svcRender *templatemodels.ServiceRender, defaultValues string, images []string, fullValues bool) (string, error) { serviceName := productSvc.ServiceName var targetContainers []*commonmodels.Container @@ -164,18 +168,18 @@ func GeneMergedValues(productSvc *commonmodels.ProductService, svcRender *templa targetChart := svcRender - replaceValuesMaps := make([]map[string]interface{}, 0) + imageValuesMaps := make([]map[string]interface{}, 0) for _, targetContainer := range targetContainers { // prepare image replace info replaceValuesMap, err := commonutil.AssignImageData(targetContainer.Image, commonutil.GetValidMatchData(targetContainer.ImagePath)) if err != nil { return "", fmt.Errorf("failed to pase image uri %s/%s, err %s", productSvc.ProductName, serviceName, err.Error()) } - replaceValuesMaps = append(replaceValuesMaps, replaceValuesMap) + imageValuesMaps = append(imageValuesMaps, replaceValuesMap) } imageKVS := make([]*helmtool.KV, 0) - for _, imageSecs := range replaceValuesMaps { + for _, imageSecs := range imageValuesMaps { for key, value := range imageSecs { imageKVS = append(imageKVS, &helmtool.KV{ Key: key, @@ -185,7 +189,7 @@ func GeneMergedValues(productSvc *commonmodels.ProductService, svcRender *templa } // replace image into service's values.yaml - replacedValuesYaml, err := commonutil.ReplaceImage(targetChart.ValuesYaml, replaceValuesMaps...) + replacedValuesYaml, err := commonutil.ReplaceImage(targetChart.ValuesYaml, imageValuesMaps...) if err != nil { return "", fmt.Errorf("failed to replace image uri %s/%s, err %s", productSvc.ProductName, serviceName, err.Error()) diff --git a/pkg/microservice/aslan/core/environment/service/environment.go b/pkg/microservice/aslan/core/environment/service/environment.go index 17087e3832..4e624b1531 100644 --- a/pkg/microservice/aslan/core/environment/service/environment.go +++ b/pkg/microservice/aslan/core/environment/service/environment.go @@ -1504,7 +1504,7 @@ func GetAffectedServices(productName, envName string, arg *K8sRendersetArg, log func GeneEstimatedValues(productName, envName, serviceOrReleaseName, scene, format string, arg *EstimateValuesArg, isHelmChartDeploy bool, log *zap.SugaredLogger) (interface{}, error) { var ( productSvc *commonmodels.ProductService - latestSvc *commonmodels.Service + tmplSvc *commonmodels.Service productInfo *commonmodels.Product err error ) @@ -1512,9 +1512,9 @@ func GeneEstimatedValues(productName, envName, serviceOrReleaseName, scene, form switch scene { case usageScenarioCreateEnv: productInfo = &commonmodels.Product{} - productSvc, latestSvc, err = prepareEstimateDataForEnvCreation(productName, serviceOrReleaseName, arg.Production, isHelmChartDeploy, log) + productSvc, tmplSvc, err = prepareEstimateDataForEnvCreation(productName, serviceOrReleaseName, arg.Production, isHelmChartDeploy, log) default: - productSvc, latestSvc, productInfo, err = prepareEstimateDataForEnvUpdate(productName, envName, serviceOrReleaseName, scene, arg.Production, isHelmChartDeploy, log) + productSvc, tmplSvc, productInfo, err = prepareEstimateDataForEnvUpdate(productName, envName, serviceOrReleaseName, scene, arg.Production, isHelmChartDeploy, log) } if err != nil { @@ -1532,15 +1532,6 @@ func GeneEstimatedValues(productName, envName, serviceOrReleaseName, scene, form images := make([]string, 0) - curUsedSvc, err := repository.QueryTemplateService(&commonrepo.ServiceFindOption{ - ServiceName: productSvc.ServiceName, - Revision: productSvc.Revision, - ProductName: productSvc.ProductName, - }, arg.Production) - if err != nil { - curUsedSvc = nil - } - mergedValues := "" if isHelmChartDeploy { chartRepo, err := commonrepo.NewHelmRepoColl().Find(&commonrepo.HelmRepoFindOption{RepoName: arg.ChartRepo}) @@ -1563,7 +1554,7 @@ func GeneEstimatedValues(productName, envName, serviceOrReleaseName, scene, form return nil, e.ErrUpdateRenderSet.AddDesc(fmt.Sprintf("failed to merge override values, err %s", err)) } } else { - containers := kube.CalculateContainer(productSvc, curUsedSvc, latestSvc.Containers, productInfo) + containers := kube.CalculateContainer(productSvc, tmplSvc, tmplSvc.Containers, productInfo) for _, container := range containers { images = append(images, container.Image) } diff --git a/pkg/microservice/aslan/core/service/service/helm.go b/pkg/microservice/aslan/core/service/service/helm.go index b9de6d5724..844fd2841d 100644 --- a/pkg/microservice/aslan/core/service/service/helm.go +++ b/pkg/microservice/aslan/core/service/service/helm.go @@ -835,7 +835,6 @@ func CreateOrUpdateHelmServiceFromRepo(projectName string, args *HelmServiceCrea filePaths = createFromRepo.Paths base = path.Join(config.S3StoragePath(), createFromRepo.Repo) - helmRenderCharts := make([]*templatemodels.ServiceRender, 0, len(filePaths)) var wg wait.Group var mux sync.RWMutex var serviceList []*commonmodels.Service @@ -962,12 +961,6 @@ func CreateOrUpdateHelmServiceFromRepo(projectName string, args *HelmServiceCrea return } serviceList = append(serviceList, svc) - - helmRenderCharts = append(helmRenderCharts, &templatemodels.ServiceRender{ - ServiceName: serviceName, - ChartVersion: svc.HelmChart.Version, - ValuesYaml: svc.HelmChart.ValuesYaml, - }) }) } diff --git a/pkg/microservice/aslan/core/workflow/service/workflow/workflow_v4.go b/pkg/microservice/aslan/core/workflow/service/workflow/workflow_v4.go index 2ca4b883e9..1fdfa26bf2 100644 --- a/pkg/microservice/aslan/core/workflow/service/workflow/workflow_v4.go +++ b/pkg/microservice/aslan/core/workflow/service/workflow/workflow_v4.go @@ -2349,17 +2349,24 @@ func CompareHelmServiceYamlInEnv(serviceName, variableYaml, envName, projectName if isHelmChartDeploy { currentYaml := "" + latestYaml := "" chartInfo := prod.GetChartDeployRenderMap()[serviceName] if chartInfo != nil { currentYaml, err = helmtool.MergeOverrideValues("", "", chartInfo.GetOverrideYaml(), chartInfo.OverrideValues, nil) if err != nil { return nil, fmt.Errorf("failed to merge override values, err: %s", err) } + + latestYaml, err = helmtool.MergeOverrideValues("", "", variableYaml, chartInfo.OverrideValues, nil) + if err != nil { + return nil, fmt.Errorf("failed to merge override values, err: %s", err) + } } currentYaml = strings.TrimSuffix(currentYaml, "\n") + latestYaml = strings.TrimSuffix(latestYaml, "\n") return &GetHelmValuesDifferenceResp{ Current: currentYaml, - Latest: variableYaml, + Latest: latestYaml, }, nil }