Skip to content

Commit

Permalink
Add attachment feature that supports attaching files to application m…
Browse files Browse the repository at this point in the history
…anifests (#4289)

* Add attachment feature

* Update detectors

* Lint

* Refactor
  • Loading branch information
khanhtc1202 authored Mar 30, 2023
1 parent 782c13b commit e234123
Show file tree
Hide file tree
Showing 9 changed files with 380 additions and 22 deletions.
12 changes: 10 additions & 2 deletions pkg/app/piped/deploysource/deploysource.go
Original file line number Diff line number Diff line change
Expand Up @@ -23,7 +23,7 @@ import (
"path/filepath"
"sync"

"github.com/pipe-cd/pipecd/pkg/app/piped/sourcedecrypter"
"github.com/pipe-cd/pipecd/pkg/app/piped/sourceprocesser"
"github.com/pipe-cd/pipecd/pkg/config"
"github.com/pipe-cd/pipecd/pkg/model"
)
Expand Down Expand Up @@ -173,13 +173,21 @@ func (p *provider) prepare(ctx context.Context, lw io.Writer) (*DeploySource, er

// Decrypt the sealed secrets if needed.
if gac.Encryption != nil && p.secretDecrypter != nil && len(gac.Encryption.DecryptionTargets) > 0 {
if err := sourcedecrypter.DecryptSecrets(appDir, *gac.Encryption, p.secretDecrypter); err != nil {
if err := sourceprocesser.DecryptSecrets(appDir, *gac.Encryption, p.secretDecrypter); err != nil {
fmt.Fprintf(lw, "Unable to decrypt the secrets (%v)\n", err)
return nil, err
}
fmt.Fprintf(lw, "Successfully decrypted secrets: %v\n", gac.Encryption.DecryptionTargets)
}

if gac.Attachment != nil && len(gac.Attachment.Targets) > 0 {
if err := sourceprocesser.AttachData(appDir, *gac.Attachment); err != nil {
fmt.Fprintf(lw, "Unable to attach the data (%v)\n", err)
return nil, err
}
fmt.Fprintf(lw, "Successfully attached data: %v\n", gac.Attachment.Targets)
}

return &DeploySource{
RepoDir: repoDir,
AppDir: appDir,
Expand Down
26 changes: 20 additions & 6 deletions pkg/app/piped/driftdetector/cloudrun/detector.go
Original file line number Diff line number Diff line change
Expand Up @@ -26,7 +26,7 @@ import (

"github.com/pipe-cd/pipecd/pkg/app/piped/livestatestore/cloudrun"
provider "github.com/pipe-cd/pipecd/pkg/app/piped/platformprovider/cloudrun"
"github.com/pipe-cd/pipecd/pkg/app/piped/sourcedecrypter"
"github.com/pipe-cd/pipecd/pkg/app/piped/sourceprocesser"
"github.com/pipe-cd/pipecd/pkg/cache"
"github.com/pipe-cd/pipecd/pkg/config"
"github.com/pipe-cd/pipecd/pkg/diff"
Expand Down Expand Up @@ -246,10 +246,15 @@ func (d *detector) loadHeadServiceManifest(app *model.Application, repo git.Repo
return provider.ServiceManifest{}, fmt.Errorf("unsupport application kind %s", cfg.Kind)
}

if d.secretDecrypter != nil && gds.Encryption != nil {
// We have to copy repository into another directory because
// decrypting the sealed secrets might change the git repository.
dir, err := os.MkdirTemp("", "detector-git-decrypt")
var (
encryptionUsed = d.secretDecrypter != nil && gds.Encryption != nil
attachmentUsed = gds.Attachment != nil
)

// We have to copy repository into another directory because
// decrypting the sealed secrets or attaching files might change the git repository.
if attachmentUsed || encryptionUsed {
dir, err := os.MkdirTemp("", "detector-git-processing")
if err != nil {
return provider.ServiceManifest{}, fmt.Errorf("failed to prepare a temporary directory for git repository (%w)", err)
}
Expand All @@ -261,11 +266,20 @@ func (d *detector) loadHeadServiceManifest(app *model.Application, repo git.Repo
}
repoDir := repo.GetPath()
appDir = filepath.Join(repoDir, app.GitPath.Path)
}

if err := sourcedecrypter.DecryptSecrets(appDir, *gds.Encryption, d.secretDecrypter); err != nil {
// Decrypting secrets to manifests.
if encryptionUsed {
if err := sourceprocesser.DecryptSecrets(appDir, *gds.Encryption, d.secretDecrypter); err != nil {
return provider.ServiceManifest{}, fmt.Errorf("failed to decrypt secrets (%w)", err)
}
}
// Then attaching configurated files to manifests.
if attachmentUsed {
if err := sourceprocesser.AttachData(appDir, *gds.Attachment); err != nil {
return provider.ServiceManifest{}, fmt.Errorf("failed to attach files (%w)", err)
}
}

var manifestFile string
if cfg.CloudRunApplicationSpec != nil {
Expand Down
26 changes: 20 additions & 6 deletions pkg/app/piped/driftdetector/kubernetes/detector.go
Original file line number Diff line number Diff line change
Expand Up @@ -26,7 +26,7 @@ import (

"github.com/pipe-cd/pipecd/pkg/app/piped/livestatestore/kubernetes"
provider "github.com/pipe-cd/pipecd/pkg/app/piped/platformprovider/kubernetes"
"github.com/pipe-cd/pipecd/pkg/app/piped/sourcedecrypter"
"github.com/pipe-cd/pipecd/pkg/app/piped/sourceprocesser"
"github.com/pipe-cd/pipecd/pkg/cache"
"github.com/pipe-cd/pipecd/pkg/config"
"github.com/pipe-cd/pipecd/pkg/diff"
Expand Down Expand Up @@ -245,10 +245,15 @@ func (d *detector) loadHeadManifests(ctx context.Context, app *model.Application
return nil, fmt.Errorf("unsupport application kind %s", cfg.Kind)
}

if d.secretDecrypter != nil && gds.Encryption != nil {
// We have to copy repository into another directory because
// decrypting the sealed secrets might change the git repository.
dir, err := os.MkdirTemp("", "detector-git-decrypt")
var (
encryptionUsed = d.secretDecrypter != nil && gds.Encryption != nil
attachmentUsed = gds.Attachment != nil
)

// We have to copy repository into another directory because
// decrypting the sealed secrets or attaching files might change the git repository.
if attachmentUsed || encryptionUsed {
dir, err := os.MkdirTemp("", "detector-git-processing")
if err != nil {
return nil, fmt.Errorf("failed to prepare a temporary directory for git repository (%w)", err)
}
Expand All @@ -260,11 +265,20 @@ func (d *detector) loadHeadManifests(ctx context.Context, app *model.Application
}
repoDir = repo.GetPath()
appDir = filepath.Join(repoDir, app.GitPath.Path)
}

if err := sourcedecrypter.DecryptSecrets(appDir, *gds.Encryption, d.secretDecrypter); err != nil {
// Decrypting secrets to manifests.
if encryptionUsed {
if err := sourceprocesser.DecryptSecrets(appDir, *gds.Encryption, d.secretDecrypter); err != nil {
return nil, fmt.Errorf("failed to decrypt secrets (%w)", err)
}
}
// Then attaching configurated files to manifests.
if attachmentUsed {
if err := sourceprocesser.AttachData(appDir, *gds.Attachment); err != nil {
return nil, fmt.Errorf("failed to attach files (%w)", err)
}
}

loader := provider.NewLoader(app.Name, appDir, repoDir, app.GitPath.ConfigFilename, cfg.KubernetesApplicationSpec.Input, d.gitClient, d.logger)
manifests, err = loader.LoadManifests(ctx)
Expand Down
26 changes: 20 additions & 6 deletions pkg/app/piped/driftdetector/terraform/detector.go
Original file line number Diff line number Diff line change
Expand Up @@ -27,7 +27,7 @@ import (

"github.com/pipe-cd/pipecd/pkg/app/piped/livestatestore/terraform"
provider "github.com/pipe-cd/pipecd/pkg/app/piped/platformprovider/terraform"
"github.com/pipe-cd/pipecd/pkg/app/piped/sourcedecrypter"
"github.com/pipe-cd/pipecd/pkg/app/piped/sourceprocesser"
"github.com/pipe-cd/pipecd/pkg/app/piped/toolregistry"
"github.com/pipe-cd/pipecd/pkg/cache"
"github.com/pipe-cd/pipecd/pkg/config"
Expand Down Expand Up @@ -190,10 +190,15 @@ func (d *detector) checkApplication(ctx context.Context, app *model.Application,
return fmt.Errorf("unsupport application kind %s", cfg.Kind)
}

if d.secretDecrypter != nil && gds.Encryption != nil {
// We have to copy repository into another directory because
// decrypting the sealed secrets might change the git repository.
dir, err := os.MkdirTemp("", "detector-git-decrypt")
var (
encryptionUsed = d.secretDecrypter != nil && gds.Encryption != nil
attachmentUsed = gds.Attachment != nil
)

// We have to copy repository into another directory because
// decrypting the sealed secrets or attaching files might change the git repository.
if attachmentUsed || encryptionUsed {
dir, err := os.MkdirTemp("", "detector-git-processing")
if err != nil {
return fmt.Errorf("failed to prepare a temporary directory for git repository (%w)", err)
}
Expand All @@ -205,11 +210,20 @@ func (d *detector) checkApplication(ctx context.Context, app *model.Application,
}
repoDir = repo.GetPath()
appDir = filepath.Join(repoDir, app.GitPath.Path)
}

if err := sourcedecrypter.DecryptSecrets(appDir, *gds.Encryption, d.secretDecrypter); err != nil {
// Decrypting secrets to manifests.
if encryptionUsed {
if err := sourceprocesser.DecryptSecrets(appDir, *gds.Encryption, d.secretDecrypter); err != nil {
return fmt.Errorf("failed to decrypt secrets (%w)", err)
}
}
// Then attaching configurated files to manifests.
if attachmentUsed {
if err := sourceprocesser.AttachData(appDir, *gds.Attachment); err != nil {
return fmt.Errorf("failed to attach files (%w)", err)
}
}

// Set up terraform
version := appCfg.Input.TerraformVersion
Expand Down
77 changes: 77 additions & 0 deletions pkg/app/piped/sourceprocesser/attacher.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,77 @@
// Copyright 2023 The PipeCD Authors.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package sourceprocesser

import (
"fmt"
"os"
"path/filepath"
"text/template"

"github.com/Masterminds/sprig/v3"

"github.com/pipe-cd/pipecd/pkg/config"
)

func AttachData(appDir string, atc config.Attachment) error {
if len(atc.Targets) == 0 {
return nil
}
if len(atc.Sources) == 0 {
return fmt.Errorf("no source data to attach")
}

src := make(map[string]string, len(atc.Sources))
for k, v := range atc.Sources {
srcPath := filepath.Join(appDir, v)
buff, err := os.ReadFile(srcPath)
if err != nil {
return fmt.Errorf("failed to read data source to attach from file %s (%w)", v, err)
}
src[k] = string(buff)
}
data := map[string](map[string]string){
"attachment": src,
}

for _, t := range atc.Targets {
targetPath := filepath.Join(appDir, t)
fileName := filepath.Base(targetPath)
tmpl := template.
New(fileName).
Funcs(sprig.TxtFuncMap()).
Option("missingkey=error")
tmpl, err := tmpl.ParseFiles(targetPath)
if err != nil {
return fmt.Errorf("failed to parse attaching target %s (%w)", t, err)
}

f, err := os.OpenFile(targetPath, os.O_WRONLY|os.O_TRUNC, 0644)
if err != nil {
return fmt.Errorf("failed to open attaching target %s (%w)", t, err)
}

if err := tmpl.Execute(f, data); err != nil {
f.Close()
return fmt.Errorf("failed to render attaching target %s (%w)", t, err)
}

if err := f.Close(); err != nil {
return fmt.Errorf("failed to close attached target %s (%w)", t, err)
}
}

return nil
}
Loading

0 comments on commit e234123

Please sign in to comment.