From bfd15bdee8059b8f5e9d7e0fb0fa45439038b845 Mon Sep 17 00:00:00 2001 From: 030 Date: Sun, 31 Dec 2023 15:22:14 +0100 Subject: [PATCH] fix: [#418] Add more examples regex usage. --- .github/workflows/docker.yml | 2 +- .trivyignore | 3 +++ cmd/n3dr/repositoriesV2.go | 1 + examples/README.md | 2 ++ internal/app/n3dr/artifactsv2/download.go | 26 ++++++++++++++++--- .../upload/maven2/snapshot/upload.go | 14 ++++++++-- .../app/n3dr/artifactsv2/upload/upload.go | 12 ++++++++- internal/app/n3dr/connection/connection.go | 2 +- 8 files changed, 53 insertions(+), 9 deletions(-) create mode 100644 .trivyignore diff --git a/.github/workflows/docker.yml b/.github/workflows/docker.yml index 29a2c48e..13fc4598 100644 --- a/.github/workflows/docker.yml +++ b/.github/workflows/docker.yml @@ -6,7 +6,7 @@ jobs: runs-on: ubuntu-20.04 steps: - uses: actions/checkout@v4.1.1 - - uses: schubergphilis/mcvs-docker-action@v0.2.1 + - uses: 030/mcvs-docker-action@18-trivyignore-validation with: dockle-accept-key: libcrypto3,libssl3 token: ${{ secrets.GITHUB_TOKEN }} diff --git a/.trivyignore b/.trivyignore new file mode 100644 index 00000000..41cf68e2 --- /dev/null +++ b/.trivyignore @@ -0,0 +1,3 @@ +# Accept the risk until 2023-01-01 +CVE-2019-14697 exp:2024-01-01 +CVE-2019-14697 exp:2023-01-01 diff --git a/cmd/n3dr/repositoriesV2.go b/cmd/n3dr/repositoriesV2.go index 9a8634fc..69888ff1 100644 --- a/cmd/n3dr/repositoriesV2.go +++ b/cmd/n3dr/repositoriesV2.go @@ -83,6 +83,7 @@ Examples: FQDN: n3drURL, HTTPS: &https, Pass: n3drPass, + Regex: regex, RepoName: n3drRepo, SkipErrors: skipErrors, User: n3drUser, diff --git a/examples/README.md b/examples/README.md index 85d8e7df..fbb42695 100644 --- a/examples/README.md +++ b/examples/README.md @@ -1,5 +1,7 @@ # Examples +regex + ## repositoriesV2 Use the [basePathPrefix](./repositoriesV2/BASE_PATH_PREFIX.md) subcommand if diff --git a/internal/app/n3dr/artifactsv2/download.go b/internal/app/n3dr/artifactsv2/download.go index 863262ae..43451328 100644 --- a/internal/app/n3dr/artifactsv2/download.go +++ b/internal/app/n3dr/artifactsv2/download.go @@ -6,6 +6,7 @@ import ( "net/http" "os" "path/filepath" + "regexp" "sync" "time" @@ -86,7 +87,7 @@ func (n *Nexus3) download(checksum, downloadedFileChecksum string, asset *models return nil } -func (n *Nexus3) downloadSingleArtifact(asset *models.AssetXO, repo string) { +func (n *Nexus3) downloadSingleArtifact(asset *models.AssetXO, repo string) error { shaType, checksum := artifacts.Checksum(asset) log.WithFields(log.Fields{ @@ -101,6 +102,17 @@ func (n *Nexus3) downloadSingleArtifact(asset *models.AssetXO, repo string) { } if !filesToBeSkipped { file := filepath.Join(n.DownloadDirName, repo, assetPath) + + // skip download of artifact if it does not match the regex + r, err := regexp.Compile(n.Regex) + if err != nil { + return err + } + if !r.MatchString(file) { + log.Debugf("file: '%s' skipped as it does not match regex: '%s'", file, n.Regex) + return nil + } + downloadedFileChecksum, err := artifacts.ChecksumLocalFile(file, shaType) if err != nil { panic(err) @@ -110,6 +122,8 @@ func (n *Nexus3) downloadSingleArtifact(asset *models.AssetXO, repo string) { panic(err) } } + + return nil } func (n *Nexus3) downloadIfChecksumMismatchLocalFile(continuationToken, repo string) error { @@ -129,13 +143,17 @@ func (n *Nexus3) downloadIfChecksumMismatchLocalFile(continuationToken, repo str for _, item := range resp.GetPayload().Items { for _, asset := range item.Assets { if n.WithoutWaitGroups || n.WithoutWaitGroupArtifacts { - n.downloadSingleArtifact(asset, repo) + if err := n.downloadSingleArtifact(asset, repo); err != nil { + return err + } } else { wg.Add(1) go func(asset *models.AssetXO) { defer wg.Done() - n.downloadSingleArtifact(asset, repo) + if err := n.downloadSingleArtifact(asset, repo); err != nil { + panic(err) + } }(asset) } } @@ -199,7 +217,7 @@ func (n *Nexus3) repository(repo *models.AbstractAPIRepository) { func (n *Nexus3) Backup() error { var wg sync.WaitGroup - cn := connection.Nexus3{BasePathPrefix: n.BasePathPrefix, FQDN: n.FQDN, DownloadDirName: n.DownloadDirName, Pass: n.Pass, User: n.User, HTTPS: n.HTTPS, DockerHost: n.DockerHost, DockerPort: n.DockerPort, DockerPortSecure: n.DockerPortSecure} + cn := connection.Nexus3{BasePathPrefix: n.BasePathPrefix, FQDN: n.FQDN, DownloadDirName: n.DownloadDirName, Pass: n.Pass, User: n.User, HTTPS: n.HTTPS, DockerHost: n.DockerHost, DockerPort: n.DockerPort, DockerPortSecure: n.DockerPortSecure, Regex: n.Regex} a := artifacts.Nexus3{Nexus3: &cn} repos, err := a.Repos() if err != nil { diff --git a/internal/app/n3dr/artifactsv2/upload/maven2/snapshot/upload.go b/internal/app/n3dr/artifactsv2/upload/maven2/snapshot/upload.go index bf27b580..30b004a5 100644 --- a/internal/app/n3dr/artifactsv2/upload/maven2/snapshot/upload.go +++ b/internal/app/n3dr/artifactsv2/upload/maven2/snapshot/upload.go @@ -14,8 +14,8 @@ import ( ) type Nexus3 struct { - HTTPS, SkipErrors bool - DownloadDirName, FQDN, Pass, RepoFormat, RepoName, User string + HTTPS, SkipErrors bool + DownloadDirName, FQDN, Pass, Regex, RepoFormat, RepoName, User string } func (n *Nexus3) statusCode(resp *http.Response) error { @@ -39,6 +39,16 @@ func (n *Nexus3) statusCode(resp *http.Response) error { } func (n *Nexus3) readRetryAndUpload(path string) error { + // skip upload of artifact if it does not match the regex + r, err := regexp.Compile(n.Regex) + if err != nil { + return err + } + if !r.MatchString(path) { + log.Debugf("file: '%s' skipped as it does not match regex: '%s'", path, n.Regex) + return nil + } + log.Debugf("reading path: '%s' and uploading it", path) f, err := os.Open(filepath.Clean(path)) if err != nil { diff --git a/internal/app/n3dr/artifactsv2/upload/upload.go b/internal/app/n3dr/artifactsv2/upload/upload.go index b9544e52..af48e099 100644 --- a/internal/app/n3dr/artifactsv2/upload/upload.go +++ b/internal/app/n3dr/artifactsv2/upload/upload.go @@ -652,6 +652,16 @@ func (n *Nexus3) ReadLocalDirAndUploadArtifacts(localDiskRepoHome, localDiskRepo return err } + // skip upload of artifact if it does not match the regex + r, err := regexp.Compile(n.Regex) + if err != nil { + return err + } + if !r.MatchString(path) { + log.Debugf("file: '%s' skipped as it does not match regex: '%s'", path, n.Regex) + return nil + } + filesToBeSkipped, err := artifacts.FilesToBeSkipped(filepath.Ext(path)) if err != nil { return err @@ -704,7 +714,7 @@ func (n *Nexus3) maven2SnapshotsUpload(localDiskRepo string) { log.Tracef("VersionPolicy: '%s'", vp) if strings.EqualFold(vp, "snapshot") { - s := snapshot.Nexus3{DownloadDirName: n.DownloadDirName, FQDN: n.FQDN, HTTPS: *n.HTTPS, Pass: n.Pass, RepoFormat: "maven2", RepoName: localDiskRepo, SkipErrors: n.SkipErrors, User: n.User} + s := snapshot.Nexus3{DownloadDirName: n.DownloadDirName, FQDN: n.FQDN, HTTPS: *n.HTTPS, Pass: n.Pass, RepoFormat: "maven2", Regex: n.Regex, RepoName: localDiskRepo, SkipErrors: n.SkipErrors, User: n.User} if err := s.Upload(); err != nil { uploaded, errRegex := regexp.MatchString("bad status: 400 Repository does not allow updating assets", err.Error()) diff --git a/internal/app/n3dr/connection/connection.go b/internal/app/n3dr/connection/connection.go index b70d96ca..b5f65138 100644 --- a/internal/app/n3dr/connection/connection.go +++ b/internal/app/n3dr/connection/connection.go @@ -9,7 +9,7 @@ import ( ) type Nexus3 struct { - AwsBucket, AwsID, AwsRegion, AwsSecret, BasePathPrefix, DockerHost, DownloadDirName, DownloadDirNameZip, Pass, RepoName, User string + AwsBucket, AwsID, AwsRegion, AwsSecret, BasePathPrefix, DockerHost, DownloadDirName, DownloadDirNameZip, Pass, Regex, RepoName, User string DockerPort int32 DockerPortSecure, SkipErrors, StrictContentTypeValidation, WithoutWaitGroups, WithoutWaitGroupArtifacts, WithoutWaitGroupRepositories, ZIP bool HTTPS *bool `validate:"required"`