Cherry-pick #5295 #5299 #5306 #5308 #5310 #5312 #5315 (#5316)

* Add event context (#5295) * Add contexts to the RegisterEventRequest Signed-off-by: Yoshiki Fujikane <[email protected]> * Add contexts to model.Event Signed-off-by: Yoshiki Fujikane <[email protected]> * Store event context in Control Plane Signed-off-by: Yoshiki Fujikane <[email protected]> * Add trailers when commiting on event watcher Signed-off-by: Yoshiki Fujikane <[email protected]> * Fix for failed build Signed-off-by: Yoshiki Fujikane <[email protected]> --------- Signed-off-by: Yoshiki Fujikane <[email protected]> Signed-off-by: pipecd-bot <[email protected]> * Add docs for pipectl event register --contexts on the event watcher usage page (#5299) * Add docs for pipectl event register --contexts on the event watcher usage page Signed-off-by: Yoshiki Fujikane <[email protected]> * Fix docs Signed-off-by: Yoshiki Fujikane <[email protected]> * Fix command Signed-off-by: Yoshiki Fujikane <[email protected]> --------- Signed-off-by: Yoshiki Fujikane <[email protected]> Signed-off-by: pipecd-bot <[email protected]> * Clone manifests not to modify original manifests (#5306) Signed-off-by: t-kikuc <[email protected]> Signed-off-by: pipecd-bot <[email protected]> * Lambda: clone manifests not to modify original manifests (#5308) Signed-off-by: t-kikuc <[email protected]> Signed-off-by: pipecd-bot <[email protected]> * Skip commit and push when no replacement happens in EventWatcher (#5310) * fix: skip push if no diff exist Signed-off-by: t-kikuc <[email protected]> * modify message Signed-off-by: t-kikuc <[email protected]> * refactor: use noChange variable Signed-off-by: t-kikuc <[email protected]> --------- Signed-off-by: t-kikuc <[email protected]> Signed-off-by: pipecd-bot <[email protected]> * Execute git clean partially when drift detection for every app is done (#5312) * Execute git clean partially when drift detection for every app is done Signed-off-by: Yoshiki Fujikane <[email protected]> * Rename to CleanPath Signed-off-by: Yoshiki Fujikane <[email protected]> * Add test for the outside dir pattern Signed-off-by: Yoshiki Fujikane <[email protected]> --------- Signed-off-by: Yoshiki Fujikane <[email protected]> Signed-off-by: pipecd-bot <[email protected]> * Update RELEASE and docs to v0.49.3 (#5315) Signed-off-by: Yoshiki Fujikane <[email protected]> Signed-off-by: pipecd-bot <[email protected]> --------- Signed-off-by: Yoshiki Fujikane <[email protected]> Signed-off-by: pipecd-bot <[email protected]> Signed-off-by: t-kikuc <[email protected]> Co-authored-by: Yoshiki Fujikane <[email protected]> Co-authored-by: Tetsuya Kikuchi <[email protected]>
pipe-cd · Nov 7, 2024 · e5fe1cf · e5fe1cf
1 parent eb98030
commit e5fe1cf
Show file tree

Hide file tree

Showing 26 changed files with 767 additions and 410 deletions.
diff --git a/RELEASE b/RELEASE
@@ -1,6 +1,6 @@
 # Generated by `make release` command.
 # DO NOT EDIT.
-tag: v0.49.2
+tag: v0.49.3
 
 releaseNoteGenerator:
   showCommitter: false

diff --git a/docs/content/en/docs-dev/user-guide/command-line-tool.md b/docs/content/en/docs-dev/user-guide/command-line-tool.md
@@ -328,6 +328,8 @@ pipectl event register \
     --data=gcr.io/pipecd/example:v0.1.0
 ```
 
+See more on [usage of Event Watcher](./event-watcher.md).
+
 ### Encrypting the data you want to use when deploying
 
 Encrypt the plaintext entered either in stdin or via the `--input-file` flag.

diff --git a/docs/content/en/docs-dev/user-guide/event-watcher.md b/docs/content/en/docs-dev/user-guide/event-watcher.md
@@ -165,6 +165,38 @@ pipectl event register \
 
 Note that it is considered a match only when labels are an exact match.
 
+### [optional] Using contexts
+
+You can also attach additional metadata to the event.
+This information can be added as a trailer to the git commit when Event Watcher using the GIT_UPDATE handler.
+This can be useful when attaching information from the source code repository to the manifest repository.
+
+For example, you can attach the source code commit link to the manifest repository.
+
+```bash
+pipectl event register \
+    --address=CONTROL_PLANE_API_ADDRESS \
+    --api-key=API_KEY \
+    --name=sample \
+    --data=gcr.io/pipecd/helloworld:v0.48.0 \
+    --contexts Source-Commit-Hash=xxxxxxx,Source-Commit-URL=https://github.com/pipe-cd/pipecd/commit/xxxxxxx
+```
+
+```bash
+# In manifest repository
+$ git show                                                                        
+commit ff46cdc9a3ce87a9a66436269251a4870ac55183 (HEAD -> main, origin/main, origin/HEAD)
+Author: ffjlabo <[email protected]>
+Date:   Wed Oct 30 16:56:36 2024 +0900
+
+    Replace values with "gcr.io/pipecd/helloworld:v0.48.0" set by Event "simple"
+
+    Source-Commit-Hash: xxxxxxx
+    Source-Commit-URL: https://github.com/pipe-cd/pipecd/commit/xxxxxxx
+```
+
+![](/images/event-watcher-contexts.png)
+
 ## Examples
 Suppose you want to update your configuration file after releasing a new Helm chart.
 

diff --git a/docs/content/en/docs-v0.49.x/user-guide/command-line-tool.md b/docs/content/en/docs-v0.49.x/user-guide/command-line-tool.md
@@ -328,6 +328,8 @@ pipectl event register \
     --data=gcr.io/pipecd/example:v0.1.0
 ```
 
+See more on [usage of Event Watcher](./event-watcher.md).
+
 ### Encrypting the data you want to use when deploying
 
 Encrypt the plaintext entered either in stdin or via the `--input-file` flag.

diff --git a/docs/content/en/docs-v0.49.x/user-guide/event-watcher.md b/docs/content/en/docs-v0.49.x/user-guide/event-watcher.md
@@ -165,6 +165,38 @@ pipectl event register \
 
 Note that it is considered a match only when labels are an exact match.
 
+### [optional] Using contexts
+
+You can also attach additional metadata to the event.
+This information can be added as a trailer to the git commit when Event Watcher using the GIT_UPDATE handler.
+This can be useful when attaching information from the source code repository to the manifest repository.
+
+For example, you can attach the source code commit link to the manifest repository.
+
+```bash
+pipectl event register \
+    --address=CONTROL_PLANE_API_ADDRESS \
+    --api-key=API_KEY \
+    --name=sample \
+    --data=gcr.io/pipecd/helloworld:v0.48.0 \
+    --contexts Source-Commit-Hash=xxxxxxx,Source-Commit-URL=https://github.com/pipe-cd/pipecd/commit/xxxxxxx
+```
+
+```bash
+# In manifest repository
+$ git show                                                                        
+commit ff46cdc9a3ce87a9a66436269251a4870ac55183 (HEAD -> main, origin/main, origin/HEAD)
+Author: ffjlabo <[email protected]>
+Date:   Wed Oct 30 16:56:36 2024 +0900
+
+    Replace values with "gcr.io/pipecd/helloworld:v0.48.0" set by Event "simple"
+
+    Source-Commit-Hash: xxxxxxx
+    Source-Commit-URL: https://github.com/pipe-cd/pipecd/commit/xxxxxxx
+```
+
+![](/images/event-watcher-contexts.png)
+
 ## Examples
 Suppose you want to update your configuration file after releasing a new Helm chart.
 

diff --git a/docs/static/images/event-watcher-contexts.png b/docs/static/images/event-watcher-contexts.png
diff --git a/pkg/app/pipectl/cmd/event/register.go b/pkg/app/pipectl/cmd/event/register.go
@@ -28,9 +28,10 @@ import (
 type register struct {
 	root *command
 
-	name   string
-	data   string
-	labels map[string]string
+	name     string
+	data     string
+	labels   map[string]string
+	contexts map[string]string
 }
 
 func newRegisterCommand(root *command) *cobra.Command {
@@ -46,6 +47,7 @@ func newRegisterCommand(root *command) *cobra.Command {
 	cmd.Flags().StringVar(&r.name, "name", r.name, "The name of event.")
 	cmd.Flags().StringVar(&r.data, "data", r.data, "The string value of event data.")
 	cmd.Flags().StringToStringVar(&r.labels, "labels", r.labels, "The list of labels for event. Format: key=value,key2=value2")
+	cmd.Flags().StringToStringVar(&r.contexts, "contexts", r.contexts, "The list of the values for the event context. Format: key=value,key2=value2")
 
 	cmd.MarkFlagRequired("name")
 	cmd.MarkFlagRequired("data")
@@ -61,9 +63,10 @@ func (r *register) run(ctx context.Context, input cli.Input) error {
 	defer cli.Close()
 
 	req := &apiservice.RegisterEventRequest{
-		Name:   r.name,
-		Data:   r.data,
-		Labels: r.labels,
+		Name:     r.name,
+		Data:     r.data,
+		Labels:   r.labels,
+		Contexts: r.contexts,
 	}
 
 	res, err := cli.RegisterEvent(ctx, req)

diff --git a/pkg/app/piped/driftdetector/ecs/detector.go b/pkg/app/piped/driftdetector/ecs/detector.go
@@ -209,11 +209,11 @@ func (d *detector) checkApplication(ctx context.Context, app *model.Application,
 	d.logger.Info(fmt.Sprintf("application %s has live ecs definition files", app.Id))
 
 	// Ignore some fields whech are not necessary or unable to detect diff.
-	ignoreParameters(liveManifests, headManifests)
+	live, head := ignoreParameters(liveManifests, headManifests)
 
 	result, err := provider.Diff(
-		liveManifests,
-		headManifests,
+		live,
+		head,
 		diff.WithEquateEmpty(),
 		diff.WithIgnoreAddingMapKeys(),
 		diff.WithCompareNumberAndNumericString(),
@@ -237,8 +237,8 @@ func (d *detector) checkApplication(ctx context.Context, app *model.Application,
 // TODO: Maybe we should check diff of following fields when not set in the head manifests in some way. Currently they are ignored:
 //   - service.PlatformVersion
 //   - service.RoleArn
-func ignoreParameters(liveManifests provider.ECSManifests, headManifests provider.ECSManifests) {
-	liveService := liveManifests.ServiceDefinition
+func ignoreParameters(liveManifests provider.ECSManifests, headManifests provider.ECSManifests) (live, head provider.ECSManifests) {
+	liveService := *liveManifests.ServiceDefinition
 	liveService.CreatedAt = nil
 	liveService.CreatedBy = nil
 	liveService.Events = nil
@@ -251,9 +251,10 @@ func ignoreParameters(liveManifests provider.ECSManifests, headManifests provide
 	liveService.TaskDefinition = nil // TODO: Find a way to compare the task definition if possible.
 	liveService.TaskSets = nil
 
-	liveTask := liveManifests.TaskDefinition
-	// When liveTask does not exist, e.g. right after the service is created.
-	if liveTask != nil {
+	var liveTask types.TaskDefinition
+	if liveManifests.TaskDefinition != nil {
+		// When liveTask does not exist, e.g. right after the service is created.
+		liveTask = *liveManifests.TaskDefinition
 		liveTask.RegisteredAt = nil
 		liveTask.RegisteredBy = nil
 		liveTask.RequiresAttributes = nil
@@ -267,7 +268,7 @@ func ignoreParameters(liveManifests provider.ECSManifests, headManifests provide
 		}
 	}
 
-	headService := headManifests.ServiceDefinition
+	headService := *headManifests.ServiceDefinition
 	if headService.PlatformVersion == nil {
 		// The LATEST platform version is used by default if PlatformVersion is not specified.
 		// See https://docs.aws.amazon.com/AmazonECS/latest/APIReference/API_CreateService.html#ECS-CreateService-request-platformVersion.
@@ -279,37 +280,43 @@ func ignoreParameters(liveManifests provider.ECSManifests, headManifests provide
 		headService.RoleArn = liveService.RoleArn
 	}
 	if headService.NetworkConfiguration != nil && headService.NetworkConfiguration.AwsvpcConfiguration != nil {
-		awsvpcCfg := headService.NetworkConfiguration.AwsvpcConfiguration
+		awsvpcCfg := *headService.NetworkConfiguration.AwsvpcConfiguration
+		awsvpcCfg.Subnets = slices.Clone(awsvpcCfg.Subnets)
 		slices.Sort(awsvpcCfg.Subnets)
 		if len(awsvpcCfg.AssignPublicIp) == 0 {
 			// AssignPublicIp is DISABLED by default.
 			// See https://docs.aws.amazon.com/AmazonECS/latest/APIReference/API_AwsVpcConfiguration.html#ECS-Type-AwsVpcConfiguration-assignPublicIp.
 			awsvpcCfg.AssignPublicIp = types.AssignPublicIpDisabled
 		}
+		headService.NetworkConfiguration = &types.NetworkConfiguration{AwsvpcConfiguration: &awsvpcCfg}
 	}
 
 	// Sort the subnets of the live service as well
 	if liveService.NetworkConfiguration != nil && liveService.NetworkConfiguration.AwsvpcConfiguration != nil {
-		awsvpcCfg := liveService.NetworkConfiguration.AwsvpcConfiguration
+		awsvpcCfg := *liveService.NetworkConfiguration.AwsvpcConfiguration
+		awsvpcCfg.Subnets = slices.Clone(awsvpcCfg.Subnets)
 		slices.Sort(awsvpcCfg.Subnets)
+		liveService.NetworkConfiguration = &types.NetworkConfiguration{AwsvpcConfiguration: &awsvpcCfg}
 	}
 
 	// TODO: In order to check diff of the tags, we need to add pipecd-managed tags and sort.
 	liveService.Tags = nil
 	headService.Tags = nil
 
-	headTask := headManifests.TaskDefinition
+	headTask := *headManifests.TaskDefinition
 	headTask.Status = types.TaskDefinitionStatusActive // If livestate's status is not ACTIVE, we should re-deploy a new task definition.
-	if liveTask != nil {
+	if liveManifests.TaskDefinition != nil {
 		headTask.Compatibilities = liveTask.Compatibilities // Users can specify Compatibilities in a task definition file, but it is not used when registering a task definition.
 	}
 
+	headTask.ContainerDefinitions = slices.Clone(headManifests.TaskDefinition.ContainerDefinitions)
 	for i := range headTask.ContainerDefinitions {
 		cd := &headTask.ContainerDefinitions[i]
 		if cd.Essential == nil {
 			// Essential is true by default. See https://docs.aws.amazon.com/AmazonECS/latest/APIReference/API_ContainerDefinition.html#ECS-Type-ContainerDefinition-es.
 			cd.Essential = aws.Bool(true)
 		}
+		cd.PortMappings = slices.Clone(cd.PortMappings)
 		for j := range cd.PortMappings {
 			pm := &cd.PortMappings[j]
 			if len(pm.Protocol) == 0 {
@@ -319,6 +326,10 @@ func ignoreParameters(liveManifests provider.ECSManifests, headManifests provide
 			pm.HostPort = nil // We ignore diff of HostPort because it has several default values.
 		}
 	}
+
+	live = provider.ECSManifests{ServiceDefinition: &liveService, TaskDefinition: &liveTask}
+	head = provider.ECSManifests{ServiceDefinition: &headService, TaskDefinition: &headTask}
+	return live, head
 }
 
 func (d *detector) loadConfigs(app *model.Application, repo git.Repo, headCommit git.Commit) (provider.ECSManifests, error) {

diff --git a/pkg/app/piped/driftdetector/ecs/detector_test.go b/pkg/app/piped/driftdetector/ecs/detector_test.go
@@ -162,17 +162,25 @@ func TestIgnoreParameters(t *testing.T) {
 		},
 	}
 
-	ignoreParameters(livestate, headManifest)
+	ignoredLive, ignoredHead := ignoreParameters(livestate, headManifest)
+
 	result, err := provider.Diff(
-		livestate,
-		headManifest,
+		ignoredLive,
+		ignoredHead,
 		diff.WithEquateEmpty(),
 		diff.WithIgnoreAddingMapKeys(),
 		diff.WithCompareNumberAndNumericString(),
 	)
 
 	assert.NoError(t, err)
 	assert.Equal(t, false, result.Diff.HasDiff())
+
+	// Check if the original manifests are not modified.
+	assert.Equal(t, []string{"1_test-subnet", "0_test-subnet"}, headManifest.ServiceDefinition.NetworkConfiguration.AwsvpcConfiguration.Subnets)
+	assert.Equal(t, []string{"1_test-sg", "0_test-sg"}, livestate.ServiceDefinition.NetworkConfiguration.AwsvpcConfiguration.SecurityGroups)
+	assert.Equal(t, 0, len(headManifest.TaskDefinition.Status))
+	assert.Nil(t, headManifest.TaskDefinition.ContainerDefinitions[1].Essential)
+	assert.Equal(t, 0, len(headManifest.TaskDefinition.ContainerDefinitions[0].PortMappings[0].Protocol))
 }
 
 func TestIgnoreAutoScalingDiff(t *testing.T) {

diff --git a/pkg/app/piped/driftdetector/kubernetes/detector.go b/pkg/app/piped/driftdetector/kubernetes/detector.go
@@ -169,6 +169,23 @@ func (d *detector) check(ctx context.Context) {
 			if err := d.checkApplication(ctx, app, gitRepo, headCommit); err != nil {
 				d.logger.Error(fmt.Sprintf("failed to check application: %s", app.Id), zap.Error(err))
 			}
+
+			// Reset the app dir to the head commit.
+			// Some tools may create temporary files locally to render manifests.
+			// The detector reuses the same located local repository and it causes unexpected behavior by reusing such temporary files.
+			// So regularly run git clean on the app dir after each detection.
+			d.logger.Info("cleaning partially cloned repository",
+				zap.String("repo-id", repoID),
+				zap.String("app-id", app.Id),
+				zap.String("app-path", app.GitPath.Path),
+			)
+			if err := gitRepo.CleanPath(ctx, app.GitPath.Path); err != nil {
+				d.logger.Error("failed to clean partially cloned repository",
+					zap.String("repo-id", repoID),
+					zap.String("app-id", app.Id),
+					zap.String("app-path", app.GitPath.Path),
+					zap.Error(err))
+			}
 		}
 	}
 }

diff --git a/pkg/app/piped/driftdetector/lambda/detector.go b/pkg/app/piped/driftdetector/lambda/detector.go
@@ -207,7 +207,12 @@ func (d *detector) checkApplication(ctx context.Context, app *model.Application,
 	}
 	d.logger.Info(fmt.Sprintf("application %s has a live function manifest", app.Id))
 
-	ignoreAndSortParameters(&headManifest.Spec)
+	clonedSpec := ignoreAndSortParameters(headManifest.Spec)
+	head := provider.FunctionManifest{
+		Kind:       headManifest.Kind,
+		APIVersion: headManifest.APIVersion,
+		Spec:       clonedSpec,
+	}
 
 	// WithIgnoreAddingMapKeys option ignores all of followings:
 	//  - default value of Architecture
@@ -216,7 +221,7 @@ func (d *detector) checkApplication(ctx context.Context, app *model.Application,
 	//  - tags added in live states, including pipecd managed tags
 	result, err := provider.Diff(
 		liveManifest,
-		headManifest,
+		head,
 		diff.WithEquateEmpty(),
 		diff.WithIgnoreAddingMapKeys(),
 		diff.WithCompareNumberAndNumericString(),
@@ -238,22 +243,31 @@ func (d *detector) checkApplication(ctx context.Context, app *model.Application,
 // sorts: (Lambda sorts them in liveSpec)
 //   - Architectures in headSpec
 //   - SubnetIDs in headSpec
-func ignoreAndSortParameters(headSpec *provider.FunctionManifestSpec) {
+func ignoreAndSortParameters(headSpec provider.FunctionManifestSpec) provider.FunctionManifestSpec {
+	cloneSpec := headSpec
 	// We cannot compare SourceCode and S3 packaging because live states do not have them.
-	headSpec.SourceCode = provider.SourceCode{}
-	headSpec.S3Bucket = ""
-	headSpec.S3Key = ""
-	headSpec.S3ObjectVersion = ""
+	cloneSpec.SourceCode = provider.SourceCode{}
+	cloneSpec.S3Bucket = ""
+	cloneSpec.S3Key = ""
+	cloneSpec.S3ObjectVersion = ""
 
 	// Architectures, Environments, SubnetIDs, and Tags are sorted in live states.
 	if len(headSpec.Architectures) > 1 {
-		sort.Slice(headSpec.Architectures, func(i, j int) bool {
-			return strings.Compare(headSpec.Architectures[i].Name, headSpec.Architectures[j].Name) < 0
+		cloneSpec.Architectures = slices.Clone(headSpec.Architectures)
+		sort.Slice(cloneSpec.Architectures, func(i, j int) bool {
+			return strings.Compare(cloneSpec.Architectures[i].Name, cloneSpec.Architectures[j].Name) < 0
 		})
 	}
 	if headSpec.VPCConfig != nil && len(headSpec.VPCConfig.SubnetIDs) > 1 {
-		slices.Sort(headSpec.VPCConfig.SubnetIDs)
+		cloneSubnets := slices.Clone(headSpec.VPCConfig.SubnetIDs)
+		slices.Sort(cloneSubnets)
+		cloneSpec.VPCConfig = &provider.VPCConfig{
+			SecurityGroupIDs: headSpec.VPCConfig.SecurityGroupIDs,
+			SubnetIDs:        cloneSubnets,
+		}
 	}
+
+	return cloneSpec
 }
 
 func (d *detector) loadHeadFunctionManifest(app *model.Application, repo git.Repo, headCommit git.Commit) (provider.FunctionManifest, error) {