fix: [#418] Add more examples regex usage.

.github/workflows/docker.yml

@@ -6,7 +6,7 @@ jobs:
     runs-on: ubuntu-20.04
     steps:
       - uses: actions/[email protected]
-      - uses: schubergphilis/mcvs-docker-action@v0.2.1
+      - uses: 030/mcvs-docker-action@18-trivyignore-validation
         with:
           dockle-accept-key: libcrypto3,libssl3
           token: ${{ secrets.GITHUB_TOKEN }}

.trivyignore

@@ -0,0 +1,3 @@
+# Accept the risk until 2023-01-01
+CVE-2019-14697 exp:2024-01-01
+CVE-2019-14697 exp:2023-01-01

cmd/n3dr/repositoriesV2.go

@@ -83,6 +83,7 @@ Examples:
 			FQDN:                         n3drURL,
 			HTTPS:                        &https,
 			Pass:                         n3drPass,
+			Regex:                        regex,
 			RepoName:                     n3drRepo,
 			SkipErrors:                   skipErrors,
 			User:                         n3drUser,

examples/README.md

@@ -1,5 +1,7 @@
 # Examples
 
+regex
+
 ## repositoriesV2
 
 Use the [basePathPrefix](./repositoriesV2/BASE_PATH_PREFIX.md) subcommand if

internal/app/n3dr/artifactsv2/download.go

@@ -6,6 +6,7 @@ import (
 	"net/http"
 	"os"
 	"path/filepath"
+	"regexp"
 	"sync"
 	"time"
 
@@ -86,7 +87,7 @@ func (n *Nexus3) download(checksum, downloadedFileChecksum string, asset *models
 	return nil
 }
 
-func (n *Nexus3) downloadSingleArtifact(asset *models.AssetXO, repo string) {
+func (n *Nexus3) downloadSingleArtifact(asset *models.AssetXO, repo string) error {
 	shaType, checksum := artifacts.Checksum(asset)
 
 	log.WithFields(log.Fields{
@@ -101,6 +102,17 @@ func (n *Nexus3) downloadSingleArtifact(asset *models.AssetXO, repo string) {
 	}
 	if !filesToBeSkipped {
 		file := filepath.Join(n.DownloadDirName, repo, assetPath)
+
+		// skip download of artifact if it does not match the regex
+		r, err := regexp.Compile(n.Regex)
+		if err != nil {
+			return err
+		}
+		if !r.MatchString(file) {
+			log.Debugf("file: '%s' skipped as it does not match regex: '%s'", file, n.Regex)
+			return nil
+		}
+
 		downloadedFileChecksum, err := artifacts.ChecksumLocalFile(file, shaType)
 		if err != nil {
 			panic(err)
@@ -110,6 +122,8 @@ func (n *Nexus3) downloadSingleArtifact(asset *models.AssetXO, repo string) {
 			panic(err)
 		}
 	}
+
+	return nil
 }
 
 func (n *Nexus3) downloadIfChecksumMismatchLocalFile(continuationToken, repo string) error {
@@ -129,13 +143,17 @@ func (n *Nexus3) downloadIfChecksumMismatchLocalFile(continuationToken, repo str
 	for _, item := range resp.GetPayload().Items {
 		for _, asset := range item.Assets {
 			if n.WithoutWaitGroups || n.WithoutWaitGroupArtifacts {
-				n.downloadSingleArtifact(asset, repo)
+				if err := n.downloadSingleArtifact(asset, repo); err != nil {
+					return err
+				}
 			} else {
 				wg.Add(1)
 				go func(asset *models.AssetXO) {
 					defer wg.Done()
 
-					n.downloadSingleArtifact(asset, repo)
+					if err := n.downloadSingleArtifact(asset, repo); err != nil {
+						panic(err)
+					}
 				}(asset)
 			}
 		}
@@ -199,7 +217,7 @@ func (n *Nexus3) repository(repo *models.AbstractAPIRepository) {
 func (n *Nexus3) Backup() error {
 	var wg sync.WaitGroup
 
-	cn := connection.Nexus3{BasePathPrefix: n.BasePathPrefix, FQDN: n.FQDN, DownloadDirName: n.DownloadDirName, Pass: n.Pass, User: n.User, HTTPS: n.HTTPS, DockerHost: n.DockerHost, DockerPort: n.DockerPort, DockerPortSecure: n.DockerPortSecure}
+	cn := connection.Nexus3{BasePathPrefix: n.BasePathPrefix, FQDN: n.FQDN, DownloadDirName: n.DownloadDirName, Pass: n.Pass, User: n.User, HTTPS: n.HTTPS, DockerHost: n.DockerHost, DockerPort: n.DockerPort, DockerPortSecure: n.DockerPortSecure, Regex: n.Regex}
 	a := artifacts.Nexus3{Nexus3: &cn}
 	repos, err := a.Repos()
 	if err != nil {

internal/app/n3dr/artifactsv2/upload/maven2/snapshot/upload.go

@@ -14,8 +14,8 @@ import (
 )
 
 type Nexus3 struct {
-	HTTPS, SkipErrors                                       bool
-	DownloadDirName, FQDN, Pass, RepoFormat, RepoName, User string
+	HTTPS, SkipErrors                                              bool
+	DownloadDirName, FQDN, Pass, Regex, RepoFormat, RepoName, User string
 }
 
 func (n *Nexus3) statusCode(resp *http.Response) error {
@@ -39,6 +39,16 @@ func (n *Nexus3) statusCode(resp *http.Response) error {
 }
 
 func (n *Nexus3) readRetryAndUpload(path string) error {
+	// skip upload of artifact if it does not match the regex
+	r, err := regexp.Compile(n.Regex)
+	if err != nil {
+		return err
+	}
+	if !r.MatchString(path) {
+		log.Debugf("file: '%s' skipped as it does not match regex: '%s'", path, n.Regex)
+		return nil
+	}
+
 	log.Debugf("reading path: '%s' and uploading it", path)
 	f, err := os.Open(filepath.Clean(path))
 	if err != nil {

internal/app/n3dr/artifactsv2/upload/upload.go

@@ -652,6 +652,16 @@ func (n *Nexus3) ReadLocalDirAndUploadArtifacts(localDiskRepoHome, localDiskRepo
 				return err
 			}
 
+			// skip upload of artifact if it does not match the regex
+			r, err := regexp.Compile(n.Regex)
+			if err != nil {
+				return err
+			}
+			if !r.MatchString(path) {
+				log.Debugf("file: '%s' skipped as it does not match regex: '%s'", path, n.Regex)
+				return nil
+			}
+
 			filesToBeSkipped, err := artifacts.FilesToBeSkipped(filepath.Ext(path))
 			if err != nil {
 				return err
@@ -704,7 +714,7 @@ func (n *Nexus3) maven2SnapshotsUpload(localDiskRepo string) {
 	log.Tracef("VersionPolicy: '%s'", vp)
 
 	if strings.EqualFold(vp, "snapshot") {
-		s := snapshot.Nexus3{DownloadDirName: n.DownloadDirName, FQDN: n.FQDN, HTTPS: *n.HTTPS, Pass: n.Pass, RepoFormat: "maven2", RepoName: localDiskRepo, SkipErrors: n.SkipErrors, User: n.User}
+		s := snapshot.Nexus3{DownloadDirName: n.DownloadDirName, FQDN: n.FQDN, HTTPS: *n.HTTPS, Pass: n.Pass, RepoFormat: "maven2", Regex: n.Regex, RepoName: localDiskRepo, SkipErrors: n.SkipErrors, User: n.User}
 
 		if err := s.Upload(); err != nil {
 			uploaded, errRegex := regexp.MatchString("bad status: 400 Repository does not allow updating assets", err.Error())

internal/app/n3dr/connection/connection.go

@@ -9,7 +9,7 @@ import (
 )
 
 type Nexus3 struct {
-	AwsBucket, AwsID, AwsRegion, AwsSecret, BasePathPrefix, DockerHost, DownloadDirName, DownloadDirNameZip, Pass, RepoName, User              string
+	AwsBucket, AwsID, AwsRegion, AwsSecret, BasePathPrefix, DockerHost, DownloadDirName, DownloadDirNameZip, Pass, Regex, RepoName, User       string
 	DockerPort                                                                                                                                 int32
 	DockerPortSecure, SkipErrors, StrictContentTypeValidation, WithoutWaitGroups, WithoutWaitGroupArtifacts, WithoutWaitGroupRepositories, ZIP bool
 	HTTPS                                                                                                                                      *bool  `validate:"required"`