diff --git a/.github/CODEOWNERS b/.github/CODEOWNERS new file mode 100644 index 00000000..67fdb57a --- /dev/null +++ b/.github/CODEOWNERS @@ -0,0 +1,6 @@ +# default reviewers +* @greenbone/vulnerability-intelligence-maintainers + +# github actions & settings +/.github/ @greenbone/devops @greenbone/vulnerability-intelligence-maintainers + diff --git a/.github/dependabot.yml b/.github/dependabot.yml new file mode 100644 index 00000000..442f93ad --- /dev/null +++ b/.github/dependabot.yml @@ -0,0 +1,23 @@ +version: 2 +updates: + - package-ecosystem: gomod + directory: "/" + schedule: + interval: weekly + time: "04:00" + groups: + go-modules: + patterns: + - "*" + allow: + - dependency-type: direct + - dependency-type: indirect + + - package-ecosystem: "github-actions" + directory: "/" + schedule: + interval: "weekly" + groups: + github-actions: + patterns: + - "*" diff --git a/.github/workflows/codeql.yml b/.github/workflows/codeql.yml new file mode 100644 index 00000000..ad03e41d --- /dev/null +++ b/.github/workflows/codeql.yml @@ -0,0 +1,43 @@ +name: "CodeQL" + +on: + push: + branches: [ main ] + pull_request: + branches: [ main ] + schedule: + - cron: '30 5 * * 0' # 5:30h on Sundays + workflow_dispatch: + +jobs: + analyze: + name: Analyze + runs-on: ubuntu-latest + permissions: + actions: read + contents: read + security-events: write + + strategy: + fail-fast: false + matrix: + language: [ 'go' ] + + steps: + - name: Checkout repository + uses: actions/checkout@v4 + - name: Setup Go + uses: actions/setup-go@v5 + with: + go-version-file: './go.mod' + + - name: Initialize CodeQL + uses: github/codeql-action/init@v3 + with: + languages: ${{ matrix.language }} + queries: security-and-quality + + - name: Perform CodeQL Analysis + uses: github/codeql-action/analyze@v3 + with: + category: "/language:${{matrix.language}}" diff --git a/.github/workflows/conventional-commits.yml b/.github/workflows/conventional-commits.yml new file mode 100644 index 00000000..85a2b32c --- /dev/null +++ b/.github/workflows/conventional-commits.yml @@ -0,0 +1,16 @@ +name: Conventional Commits + +on: + pull_request: + +permissions: + pull-requests: write + contents: read + +jobs: + conventional-commits: + name: Conventional Commits + runs-on: ubuntu-latest + steps: + - name: Report Conventional Commits + uses: greenbone/actions/conventional-commits@v3 diff --git a/.github/workflows/dependency-review.yml b/.github/workflows/dependency-review.yml new file mode 100644 index 00000000..36afcc32 --- /dev/null +++ b/.github/workflows/dependency-review.yml @@ -0,0 +1,12 @@ +name: 'Dependency Review' +on: [pull_request] + +permissions: + contents: read + +jobs: + dependency-review: + runs-on: ubuntu-latest + steps: + - name: 'Dependency Review' + uses: greenbone/actions/dependency-review@v3 diff --git a/.github/workflows/generate-markdown.yml b/.github/workflows/generate-markdown.yml index a59c9444..a8b2dcc9 100644 --- a/.github/workflows/generate-markdown.yml +++ b/.github/workflows/generate-markdown.yml @@ -13,8 +13,8 @@ jobs: auto-update-readme: runs-on: ubuntu-latest steps: - - uses: actions/checkout@v2 + - uses: actions/checkout@v4 - name: Markdown autodocs - uses: dineshsonachalam/markdown-autodocs@v1.0.4 + uses: dineshsonachalam/markdown-autodocs@v1.0.7 with: output_file_paths: '[./README.md, ./docs/*.md]' diff --git a/.github/workflows/go.yml b/.github/workflows/go.yml index 95ee8c75..c7957cae 100644 --- a/.github/workflows/go.yml +++ b/.github/workflows/go.yml @@ -2,11 +2,10 @@ name: Go on: push: - paths: - - "**.go" + branches: + - main pull_request: - paths: - - "**.go" + workflow_dispatch: jobs: build: @@ -15,26 +14,12 @@ jobs: - uses: actions/checkout@v4 - name: Set up Go - uses: actions/setup-go@v4 + uses: actions/setup-go@v5 with: go-version: 'stable' - name: Build run: go build -v ./cmd/... - - name: vet - run: go vet ./... - - - name: gofmt - uses: Jerome1337/gofmt-action@v1.0.4 - with: - gofmt-flags: "-l -d" - - - name: golint - uses: Jerome1337/golint-action@v1.0.2 - - - name: Revive Action - uses: morphy2k/revive-action@v2.5.1 - - name: Tests run: go test -v ./... diff --git a/.github/workflows/go_legacy.yml b/.github/workflows/go_legacy.yml deleted file mode 100644 index a86368dc..00000000 --- a/.github/workflows/go_legacy.yml +++ /dev/null @@ -1,26 +0,0 @@ -name: Go - -on: - push: - paths: - - "**.go" - pull_request: - paths: - - "**.go" - -jobs: - build: - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v4 - - - name: Set up Go - uses: actions/setup-go@v4 - with: - go-version: 'oldstable' - - - name: Build - run: go build -v ./cmd/... - - - name: Tests - run: go test -v ./... diff --git a/.github/workflows/govulncheck.yml b/.github/workflows/govulncheck.yml new file mode 100644 index 00000000..a8f5eac7 --- /dev/null +++ b/.github/workflows/govulncheck.yml @@ -0,0 +1,19 @@ +name: govulncheck + +# check for vulnerabilities using `govulncheck`, compared to dependabot it only alerts if the affected code is actually called + +on: + pull_request: # make sure there is no vulnerability added with a new feature + schedule: # check if used code of existing dependencies is vulnerable + - cron: '37 4 * * *' # daily 4:37h + workflow_dispatch: # on demand + +jobs: + govulncheck: + runs-on: ubuntu-latest + name: Run govulncheck + steps: + - id: govulncheck + uses: golang/govulncheck-action@v1 + with: + go-version-file: ./go.mod \ No newline at end of file diff --git a/.github/workflows/itest.yml b/.github/workflows/itest.yml index eff11c24..435f62fe 100644 --- a/.github/workflows/itest.yml +++ b/.github/workflows/itest.yml @@ -7,17 +7,17 @@ jobs: steps: - name: Set up Go - uses: actions/setup-go@v3 + uses: actions/setup-go@v5 with: go-version: 1.21.0 - name: Set up Node.js - uses: actions/setup-node@v3 + uses: actions/setup-node@v4 with: node-version: 16 - name: Checkout - uses: actions/checkout@v3 + uses: actions/checkout@v4 - name: Execute the scripts run: | @@ -36,7 +36,7 @@ jobs: shell: bash - name: Upload test results - uses: actions/upload-artifact@v3 + uses: actions/upload-artifact@v4 with: name: checker-results path: | diff --git a/.github/workflows/linting.yml b/.github/workflows/linting.yml new file mode 100644 index 00000000..02152108 --- /dev/null +++ b/.github/workflows/linting.yml @@ -0,0 +1,16 @@ +name: Lint + +on: + push: + branches: + - main + pull_request: + +jobs: + lint: + name: lint + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + - uses: actions/setup-go@v5 + - uses: golangci/golangci-lint-action@v6 diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 739f45c1..9cd6df07 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -1,28 +1,27 @@ -name: Publish Go binaries to github release +name: Release +# with pontos on: - release: - types: [created] + pull_request: + types: [closed] + workflow_dispatch: + inputs: + release-type: + type: choice + description: What kind of release do you want to do (pontos --release-type argument)? + options: + - patch + - minor + - major + release-version: + type: string + description: Set an explicit version, that will overwrite release-type. Fails if version is not compliant. jobs: - releases-matrix: - name: Release Go binaries - runs-on: ubuntu-20.04 - steps: - - name: Checkout - uses: actions/checkout@v3 - - - name: Set up Go - uses: actions/setup-go@v3 - with: - go-version: '^1.21.0' - - - name: Build - run: make dist - - - name: Upload release assets - uses: softprops/action-gh-release@v1 - with: - files: | - dist/csaf_distribution-*.zip - dist/csaf_distribution-*.tar.gz + release: + name: csaf_distribution + uses: greenbone/workflows/.github/workflows/release-generic.yml@main + with: + release-type: ${{ inputs.release-type }} + release-version: ${{ inputs.release-version }} + secrets: inherit diff --git a/cmd/csaf_aggregator/config.go b/cmd/csaf_aggregator/config.go index b73286c3..29ec742c 100644 --- a/cmd/csaf_aggregator/config.go +++ b/cmd/csaf_aggregator/config.go @@ -23,8 +23,8 @@ import ( "github.com/csaf-poc/csaf_distribution/v3/csaf" "github.com/csaf-poc/csaf_distribution/v3/internal/certs" "github.com/csaf-poc/csaf_distribution/v3/internal/filter" - "github.com/csaf-poc/csaf_distribution/v3/internal/models" - "github.com/csaf-poc/csaf_distribution/v3/internal/options" + "github.com/csaf-poc/csaf_distribution/v3/pkg/models" + "github.com/csaf-poc/csaf_distribution/v3/pkg/options" "github.com/csaf-poc/csaf_distribution/v3/util" "golang.org/x/time/rate" ) diff --git a/cmd/csaf_aggregator/main.go b/cmd/csaf_aggregator/main.go index 39c10518..68907d30 100644 --- a/cmd/csaf_aggregator/main.go +++ b/cmd/csaf_aggregator/main.go @@ -15,8 +15,7 @@ import ( "os" "path/filepath" - "github.com/csaf-poc/csaf_distribution/v3/internal/options" - + "github.com/csaf-poc/csaf_distribution/v3/pkg/options" "github.com/gofrs/flock" ) diff --git a/cmd/csaf_checker/config.go b/cmd/csaf_checker/config.go index ac9ce62b..a31910d5 100644 --- a/cmd/csaf_checker/config.go +++ b/cmd/csaf_checker/config.go @@ -15,8 +15,8 @@ import ( "github.com/csaf-poc/csaf_distribution/v3/internal/certs" "github.com/csaf-poc/csaf_distribution/v3/internal/filter" - "github.com/csaf-poc/csaf_distribution/v3/internal/models" - "github.com/csaf-poc/csaf_distribution/v3/internal/options" + "github.com/csaf-poc/csaf_distribution/v3/pkg/models" + "github.com/csaf-poc/csaf_distribution/v3/pkg/options" ) type outputFormat string diff --git a/cmd/csaf_checker/main.go b/cmd/csaf_checker/main.go index 752fdf8e..920a59be 100644 --- a/cmd/csaf_checker/main.go +++ b/cmd/csaf_checker/main.go @@ -12,7 +12,7 @@ package main import ( "log" - "github.com/csaf-poc/csaf_distribution/v3/internal/options" + "github.com/csaf-poc/csaf_distribution/v3/pkg/options" ) // run uses a processor to check all the given domains or direct urls diff --git a/cmd/csaf_checker/report.go b/cmd/csaf_checker/report.go index 9b5251b0..5d230f7f 100644 --- a/cmd/csaf_checker/report.go +++ b/cmd/csaf_checker/report.go @@ -19,7 +19,7 @@ import ( "time" "github.com/csaf-poc/csaf_distribution/v3/csaf" - "github.com/csaf-poc/csaf_distribution/v3/internal/models" + "github.com/csaf-poc/csaf_distribution/v3/pkg/models" ) // MessageType is the kind of the message. diff --git a/cmd/csaf_downloader/config.go b/cmd/csaf_downloader/config.go index dcfc0900..1448ecc8 100644 --- a/cmd/csaf_downloader/config.go +++ b/cmd/csaf_downloader/config.go @@ -6,7 +6,7 @@ // SPDX-FileCopyrightText: 2022 German Federal Office for Information Security (BSI) // Software-Engineering: 2022 Intevation GmbH -package main +package csaf_downloader import ( "crypto/tls" @@ -21,27 +21,27 @@ import ( "github.com/csaf-poc/csaf_distribution/v3/internal/certs" "github.com/csaf-poc/csaf_distribution/v3/internal/filter" - "github.com/csaf-poc/csaf_distribution/v3/internal/models" - "github.com/csaf-poc/csaf_distribution/v3/internal/options" + "github.com/csaf-poc/csaf_distribution/v3/pkg/models" + "github.com/csaf-poc/csaf_distribution/v3/pkg/options" ) const ( defaultWorker = 2 defaultPreset = "mandatory" defaultForwardQueue = 5 - defaultValidationMode = validationStrict + defaultValidationMode = ValidationStrict defaultLogFile = "downloader.log" defaultLogLevel = slog.LevelInfo ) -type validationMode string +type ValidationMode string const ( - validationStrict = validationMode("strict") - validationUnsafe = validationMode("unsafe") + ValidationStrict = ValidationMode("strict") + ValidationUnsafe = ValidationMode("unsafe") ) -type config struct { +type Config struct { Directory string `short:"d" long:"directory" description:"DIRectory to store the downloaded files in" value-name:"DIR" toml:"directory"` Insecure bool `long:"insecure" description:"Do not check TLS certificates from provider" toml:"insecure"` IgnoreSignatureCheck bool `long:"ignore_sigcheck" description:"Ignore signature check results, just warn on mismatch" toml:"ignore_sigcheck"` @@ -64,7 +64,7 @@ type config struct { RemoteValidatorPresets []string `long:"validator_preset" description:"One or more PRESETS to validate remotely" value-name:"PRESETS" toml:"validator_preset"` //lint:ignore SA5008 We are using choice twice: strict, unsafe. - ValidationMode validationMode `long:"validation_mode" short:"m" choice:"strict" choice:"unsafe" value-name:"MODE" description:"MODE how strict the validation is" toml:"validation_mode"` + ValidationMode ValidationMode `long:"validation_mode" short:"m" choice:"strict" choice:"unsafe" value-name:"MODE" description:"MODE how strict the validation is" toml:"validation_mode"` ForwardURL string `long:"forward_url" description:"URL of HTTP endpoint to forward downloads to" value-name:"URL" toml:"forward_url"` ForwardHeader http.Header `long:"forward_header" description:"One or more extra HTTP header fields used by forwarding" toml:"forward_header"` @@ -77,8 +77,10 @@ type config struct { Config string `short:"c" long:"config" description:"Path to config TOML file" value-name:"TOML-FILE" toml:"-"` - clientCerts []tls.Certificate + ClientCerts []tls.Certificate ignorePattern filter.PatternMatcher + + ForwardChannel bool // forward the csafs via a channel (is not meant to be set via command line) } // configPaths are the potential file locations of the config file. @@ -88,18 +90,18 @@ var configPaths = []string{ "csaf_downloader.toml", } -// parseArgsConfig parses the command line and if need a config file. -func parseArgsConfig() ([]string, *config, error) { +// ParseArgsConfig parses the command line and if need a config file. +func ParseArgsConfig() ([]string, *Config, error) { var ( logFile = defaultLogFile logLevel = &options.LogLevel{Level: defaultLogLevel} ) - p := options.Parser[config]{ + p := options.Parser[Config]{ DefaultConfigLocations: configPaths, - ConfigLocation: func(cfg *config) string { return cfg.Config }, + ConfigLocation: func(cfg *Config) string { return cfg.Config }, Usage: "[OPTIONS] domain...", - HasVersion: func(cfg *config) bool { return cfg.Version }, - SetDefaults: func(cfg *config) { + HasVersion: func(cfg *Config) bool { return cfg.Version }, + SetDefaults: func(cfg *Config) { cfg.Worker = defaultWorker cfg.RemoteValidatorPresets = []string{defaultPreset} cfg.ValidationMode = defaultValidationMode @@ -108,7 +110,7 @@ func parseArgsConfig() ([]string, *config, error) { cfg.LogLevel = logLevel }, // Re-establish default values if not set. - EnsureDefaults: func(cfg *config) { + EnsureDefaults: func(cfg *Config) { if cfg.Worker == 0 { cfg.Worker = defaultWorker } @@ -116,9 +118,9 @@ func parseArgsConfig() ([]string, *config, error) { cfg.RemoteValidatorPresets = []string{defaultPreset} } switch cfg.ValidationMode { - case validationStrict, validationUnsafe: + case ValidationStrict, ValidationUnsafe: default: - cfg.ValidationMode = validationStrict + cfg.ValidationMode = ValidationStrict } if cfg.LogFile == nil { cfg.LogFile = &logFile @@ -132,9 +134,9 @@ func parseArgsConfig() ([]string, *config, error) { } // UnmarshalText implements [encoding.TextUnmarshaler]. -func (vm *validationMode) UnmarshalText(text []byte) error { - switch m := validationMode(text); m { - case validationStrict, validationUnsafe: +func (vm *ValidationMode) UnmarshalText(text []byte) error { + switch m := ValidationMode(text); m { + case ValidationStrict, ValidationUnsafe: *vm = m default: return fmt.Errorf(`invalid value %q (expected "strict" or "unsafe)"`, m) @@ -143,8 +145,8 @@ func (vm *validationMode) UnmarshalText(text []byte) error { } // UnmarshalFlag implements [flags.UnmarshalFlag]. -func (vm *validationMode) UnmarshalFlag(value string) error { - var v validationMode +func (vm *ValidationMode) UnmarshalFlag(value string) error { + var v ValidationMode if err := v.UnmarshalText([]byte(value)); err != nil { return err } @@ -153,18 +155,18 @@ func (vm *validationMode) UnmarshalFlag(value string) error { } // ignoreFile returns true if the given URL should not be downloaded. -func (cfg *config) ignoreURL(u string) bool { +func (cfg *Config) ignoreURL(u string) bool { return cfg.ignorePattern.Matches(u) } // verbose is considered a log level equal or less debug. -func (cfg *config) verbose() bool { +func (cfg *Config) verbose() bool { return cfg.LogLevel.Level <= slog.LevelDebug } // prepareDirectory ensures that the working directory // exists and is setup properly. -func (cfg *config) prepareDirectory() error { +func (cfg *Config) prepareDirectory() error { // If not given use current working directory. if cfg.Directory == "" { dir, err := os.Getwd() @@ -197,8 +199,8 @@ func dropSubSeconds(_ []string, a slog.Attr) slog.Attr { return a } -// prepareLogging sets up the structured logging. -func (cfg *config) prepareLogging() error { +// PrepareLogging sets up the structured logging. +func (cfg *Config) PrepareLogging() error { var w io.Writer if cfg.LogFile == nil || *cfg.LogFile == "" { log.Println("using STDERR for logging") @@ -231,7 +233,7 @@ func (cfg *config) prepareLogging() error { } // compileIgnorePatterns compiles the configure patterns to be ignored. -func (cfg *config) compileIgnorePatterns() error { +func (cfg *Config) compileIgnorePatterns() error { pm, err := filter.NewPatternMatcher(cfg.IgnorePattern) if err != nil { return err @@ -241,23 +243,23 @@ func (cfg *config) compileIgnorePatterns() error { } // prepareCertificates loads the client side certificates used by the HTTP client. -func (cfg *config) prepareCertificates() error { +func (cfg *Config) prepareCertificates() error { cert, err := certs.LoadCertificate( cfg.ClientCert, cfg.ClientKey, cfg.ClientPassphrase) if err != nil { return err } - cfg.clientCerts = cert + cfg.ClientCerts = cert return nil } // prepare prepares internal state of a loaded configuration. -func (cfg *config) prepare() error { - for _, prepare := range []func(*config) error{ - (*config).prepareDirectory, - (*config).prepareLogging, - (*config).prepareCertificates, - (*config).compileIgnorePatterns, +func (cfg *Config) Prepare() error { + for _, prepare := range []func(*Config) error{ + (*Config).prepareDirectory, + (*Config).PrepareLogging, + (*Config).prepareCertificates, + (*Config).compileIgnorePatterns, } { if err := prepare(cfg); err != nil { return err diff --git a/cmd/csaf_downloader/downloader.go b/cmd/csaf_downloader/downloader.go index 9cef294c..e7639a51 100644 --- a/cmd/csaf_downloader/downloader.go +++ b/cmd/csaf_downloader/downloader.go @@ -6,7 +6,7 @@ // SPDX-FileCopyrightText: 2022, 2023 German Federal Office for Information Security (BSI) // Software-Engineering: 2022, 2023 Intevation GmbH -package main +package csaf_downloader import ( "bytes" @@ -15,7 +15,6 @@ import ( "crypto/sha512" "crypto/tls" "encoding/json" - "errors" "fmt" "hash" "io" @@ -34,17 +33,20 @@ import ( "golang.org/x/time/rate" "github.com/csaf-poc/csaf_distribution/v3/csaf" + "github.com/csaf-poc/csaf_distribution/v3/pkg/errs" + csafErrs "github.com/csaf-poc/csaf_distribution/v3/pkg/errs" "github.com/csaf-poc/csaf_distribution/v3/util" ) -type downloader struct { - cfg *config +type Downloader struct { + cfg *Config keys *crypto.KeyRing validator csaf.RemoteValidator - forwarder *forwarder + Forwarder *Forwarder mkdirMu sync.Mutex statsMu sync.Mutex stats stats + Csafs chan []byte } // failedValidationDir is the name of the sub folder @@ -52,7 +54,7 @@ type downloader struct { // unsafe mode. const failedValidationDir = "failed_validation" -func newDownloader(cfg *config) (*downloader, error) { +func NewDownloader(cfg *Config) (*Downloader, error) { var validator csaf.RemoteValidator @@ -70,21 +72,23 @@ func newDownloader(cfg *config) (*downloader, error) { validator = csaf.SynchronizedRemoteValidator(validator) } - return &downloader{ + return &Downloader{ cfg: cfg, validator: validator, + Csafs: make(chan []byte), }, nil } -func (d *downloader) close() { +func (d *Downloader) Close() { if d.validator != nil { d.validator.Close() d.validator = nil } + close(d.Csafs) } // addStats add stats to total stats -func (d *downloader) addStats(o *stats) { +func (d *Downloader) addStats(o *stats) { d.statsMu.Lock() defer d.statsMu.Unlock() d.stats.add(o) @@ -102,7 +106,7 @@ func logRedirect(req *http.Request, via []*http.Request) error { return nil } -func (d *downloader) httpClient() util.Client { +func (d *Downloader) httpClient() util.Client { hClient := http.Client{} @@ -115,8 +119,8 @@ func (d *downloader) httpClient() util.Client { tlsConfig.InsecureSkipVerify = true } - if len(d.cfg.clientCerts) != 0 { - tlsConfig.Certificates = d.cfg.clientCerts + if len(d.cfg.ClientCerts) != 0 { + tlsConfig.Certificates = d.cfg.ClientCerts } hClient.Transport = &http.Transport{ @@ -160,7 +164,7 @@ func httpLog(who string) func(string, string) { } } -func (d *downloader) enumerate(domain string) error { +func (d *Downloader) enumerate(domain string) error { client := d.httpClient() loader := csaf.NewProviderMetadataLoader(client) @@ -190,7 +194,7 @@ func (d *downloader) enumerate(domain string) error { return nil } -func (d *downloader) download(ctx context.Context, domain string) error { +func (d *Downloader) download(ctx context.Context, domain string) error { client := d.httpClient() loader := csaf.NewProviderMetadataLoader(client) @@ -203,7 +207,7 @@ func (d *downloader) download(ctx context.Context, domain string) error { "domain", domain, "message", lpmd.Messages[i].Message) } - return fmt.Errorf("no valid provider-metadata.json found for '%s'", domain) + return errs.ErrCsafProviderIssue{Message: fmt.Sprintf("no valid provider-metadata.json found for '%s'", domain)} } else if d.cfg.verbose() { for i := range lpmd.Messages { slog.Debug("Loading provider-metadata.json", @@ -214,7 +218,7 @@ func (d *downloader) download(ctx context.Context, domain string) error { base, err := url.Parse(lpmd.URL) if err != nil { - return fmt.Errorf("invalid URL '%s': %v", lpmd.URL, err) + return errs.ErrCsafProviderIssue{Message: fmt.Sprintf("invalid URL '%s': %v", lpmd.URL, err)} } expr := util.NewPathEval() @@ -225,7 +229,7 @@ func (d *downloader) download(ctx context.Context, domain string) error { base, expr, ); err != nil { - return err + return errs.ErrCsafProviderIssue{Message: err.Error()} } afp := csaf.NewAdvisoryFileProcessor( @@ -246,7 +250,7 @@ func (d *downloader) download(ctx context.Context, domain string) error { }) } -func (d *downloader) downloadFiles( +func (d *Downloader) downloadFiles( ctx context.Context, label csaf.TLPLabel, files []csaf.AdvisoryFile, @@ -292,10 +296,13 @@ allFiles: close(errorCh) <-errDone - return errors.Join(errs...) + if len(errs) > 0 { + return &csafErrs.CompositeErrCsafDownload{Errs: errs} + } + return nil } -func (d *downloader) loadOpenPGPKeys( +func (d *Downloader) loadOpenPGPKeys( client util.Client, doc any, base *url.URL, @@ -387,7 +394,7 @@ func (d *downloader) loadOpenPGPKeys( } // logValidationIssues logs the issues reported by the advisory schema validation. -func (d *downloader) logValidationIssues(url string, errors []string, err error) { +func (d *Downloader) logValidationIssues(url string, errors []string, err error) { if err != nil { slog.Error("Failed to validate", "url", url, @@ -407,7 +414,7 @@ func (d *downloader) logValidationIssues(url string, errors []string, err error) } } -func (d *downloader) downloadWorker( +func (d *Downloader) downloadWorker( ctx context.Context, wg *sync.WaitGroup, label csaf.TLPLabel, @@ -461,6 +468,7 @@ nextAdvisory: filename := filepath.Base(u.Path) if !util.ConformingFileName(filename) { stats.filenameFailed++ + errorCh <- csafErrs.ErrInvalidCsaf{Message: fmt.Sprintf("CSAF has non conforming filename %s", filename)} slog.Warn("Ignoring none conforming filename", "filename", filename) continue @@ -469,6 +477,7 @@ nextAdvisory: resp, err := client.Get(file.URL()) if err != nil { stats.downloadFailed++ + errorCh <- csafErrs.ErrNetwork{Message: fmt.Sprintf("can't retrieve CSAF document %s from URL %s: %v", filename, file.URL(), err)} slog.Warn("Cannot GET", "url", file.URL(), "error", err) @@ -476,6 +485,16 @@ nextAdvisory: } if resp.StatusCode != http.StatusOK { + switch { + case resp.StatusCode == http.StatusUnauthorized: + errorCh <- csafErrs.ErrInvalidCredentials{Message: fmt.Sprintf("invalid credentials to retrieve CSAF document %s at URL %s: %s", filename, file.URL(), resp.Status)} + case resp.StatusCode == http.StatusNotFound: + errorCh <- csafErrs.ErrCsafProviderIssue{Message: fmt.Sprintf("could not find CSAF document %s listed in table of content at URL %s: %s ", filename, file.URL(), resp.Status)} + case resp.StatusCode >= 500: + errorCh <- fmt.Errorf("could not retrieve CSAF document %s at URL %s: %s %w", filename, file.URL(), resp.Status, csafErrs.ErrRetryable) // mark as retryable error + default: + errorCh <- fmt.Errorf("could not retrieve CSAF document %s at URL %s: %s", filename, file.URL(), resp.Status) + } stats.downloadFailed++ slog.Warn("Cannot load", "url", file.URL(), @@ -533,6 +552,7 @@ nextAdvisory: return json.NewDecoder(tee).Decode(&doc) }(); err != nil { stats.downloadFailed++ + errorCh <- csafErrs.ErrInvalidCsaf{Message: fmt.Sprintf("CSAF document %s at URL %s is not valid json: %v", filename, file.URL(), err)} slog.Warn("Downloading failed", "url", file.URL(), "error", err) @@ -543,6 +563,7 @@ nextAdvisory: s256Check := func() error { if s256 != nil && !bytes.Equal(s256.Sum(nil), remoteSHA256) { stats.sha256Failed++ + errorCh <- csafErrs.ErrCsafProviderIssue{Message: fmt.Sprintf("SHA256 checksum of CSAF document %s at URL %s does not match", filename, file.URL())} return fmt.Errorf("SHA256 checksum of %s does not match", file.URL()) } return nil @@ -551,6 +572,7 @@ nextAdvisory: s512Check := func() error { if s512 != nil && !bytes.Equal(s512.Sum(nil), remoteSHA512) { stats.sha512Failed++ + errorCh <- csafErrs.ErrCsafProviderIssue{Message: fmt.Sprintf("SHA512 checksum of CSAF document %s at URL %s does not match", filename, file.URL())} return fmt.Errorf("SHA512 checksum of %s does not match", file.URL()) } return nil @@ -573,6 +595,7 @@ nextAdvisory: if err := d.checkSignature(data.Bytes(), sign); err != nil { if !d.cfg.IgnoreSignatureCheck { stats.signatureFailed++ + errorCh <- csafErrs.ErrCsafProviderIssue{Message: fmt.Sprintf("cannot verify signature for CSAF document %s at URL %s: %v", filename, file.URL(), err)} return fmt.Errorf("cannot verify signature for %s: %v", file.URL(), err) } } @@ -584,6 +607,11 @@ nextAdvisory: schemaCheck := func() error { if errors, err := csaf.ValidateCSAF(doc); err != nil || len(errors) > 0 { stats.schemaFailed++ + if err != nil { + errorCh <- fmt.Errorf("schema validation for CSAF document %s failed: %w", filename, err) + } else { + errorCh <- csafErrs.ErrInvalidCsaf{Message: fmt.Sprintf("CSAF document %s at URL %s does not conform to JSON schema: %v", filename, file.URL(), errors)} + } d.logValidationIssues(file.URL(), errors, err) return fmt.Errorf("schema validation for %q failed", file.URL()) } @@ -594,6 +622,7 @@ nextAdvisory: filenameCheck := func() error { if err := util.IDMatchesFilename(expr, doc, filename); err != nil { stats.filenameFailed++ + errorCh <- csafErrs.ErrInvalidCsaf{Message: fmt.Sprintf("invalid CSAF document %s at URL %s: %v", filename, file.URL(), err)} return fmt.Errorf("filename not conforming %s: %s", file.URL(), err) } return nil @@ -613,6 +642,7 @@ nextAdvisory: } if !rvr.Valid { stats.remoteFailed++ + errorCh <- csafErrs.ErrInvalidCsaf{Message: fmt.Sprintf("remote validation of CSAF document %s at URL %s failed", filename, file.URL())} return fmt.Errorf("remote validation of %q failed", file.URL()) } return nil @@ -631,7 +661,7 @@ nextAdvisory: if err := check(); err != nil { slog.Error("Validation check failed", "error", err) valStatus.update(invalidValidationStatus) - if d.cfg.ValidationMode == validationStrict { + if d.cfg.ValidationMode == ValidationStrict { continue nextAdvisory } } @@ -639,14 +669,18 @@ nextAdvisory: valStatus.update(validValidationStatus) // Send to forwarder - if d.forwarder != nil { - d.forwarder.forward( + if d.Forwarder != nil { + d.Forwarder.forward( filename, data.String(), valStatus, string(s256Data), string(s512Data)) } + if d.cfg.ForwardChannel { + d.Csafs <- data.Bytes() + } + if d.cfg.NoStore { // Do not write locally. if valStatus == validValidationStatus { @@ -713,13 +747,13 @@ nextAdvisory: } } -func (d *downloader) mkdirAll(path string, perm os.FileMode) error { +func (d *Downloader) mkdirAll(path string, perm os.FileMode) error { d.mkdirMu.Lock() defer d.mkdirMu.Unlock() return os.MkdirAll(path, perm) } -func (d *downloader) checkSignature(data []byte, sign *crypto.PGPSignature) error { +func (d *Downloader) checkSignature(data []byte, sign *crypto.PGPSignature) error { pm := crypto.NewPlainMessage(data) t := crypto.GetUnixTime() return d.keys.VerifyDetached(pm, sign, t) @@ -766,7 +800,7 @@ func loadHash(client util.Client, p string) ([]byte, []byte, error) { } // run performs the downloads for all the given domains. -func (d *downloader) run(ctx context.Context, domains []string) error { +func (d *Downloader) Run(ctx context.Context, domains []string) error { defer d.stats.log() for _, domain := range domains { if err := d.download(ctx, domain); err != nil { @@ -777,7 +811,7 @@ func (d *downloader) run(ctx context.Context, domains []string) error { } // runEnumerate performs the enumeration of PMDs for all the given domains. -func (d *downloader) runEnumerate(domains []string) error { +func (d *Downloader) RunEnumerate(domains []string) error { defer d.stats.log() for _, domain := range domains { if err := d.enumerate(domain); err != nil { diff --git a/cmd/csaf_downloader/forwarder.go b/cmd/csaf_downloader/forwarder.go index c3681eba..be92df02 100644 --- a/cmd/csaf_downloader/forwarder.go +++ b/cmd/csaf_downloader/forwarder.go @@ -6,7 +6,7 @@ // SPDX-FileCopyrightText: 2023 German Federal Office for Information Security (BSI) // Software-Engineering: 2023 Intevation GmbH -package main +package csaf_downloader import ( "bytes" @@ -44,31 +44,31 @@ func (vs *validationStatus) update(status validationStatus) { } } -// forwarder forwards downloaded advisories to a given +// Forwarder forwards downloaded advisories to a given // HTTP endpoint. -type forwarder struct { - cfg *config - cmds chan func(*forwarder) +type Forwarder struct { + cfg *Config + cmds chan func(*Forwarder) client util.Client failed int succeeded int } -// newForwarder creates a new forwarder. -func newForwarder(cfg *config) *forwarder { +// NewForwarder creates a new forwarder. +func NewForwarder(cfg *Config) *Forwarder { queue := cfg.ForwardQueue if queue < 1 { queue = 1 } - return &forwarder{ + return &Forwarder{ cfg: cfg, - cmds: make(chan func(*forwarder), queue), + cmds: make(chan func(*Forwarder), queue), } } -// run runs the forwarder. Meant to be used in a Go routine. -func (f *forwarder) run() { +// Run runs the forwarder. Meant to be used in a Go routine. +func (f *Forwarder) Run() { defer slog.Debug("forwarder done") for cmd := range f.cmds { @@ -77,13 +77,13 @@ func (f *forwarder) run() { } // close terminates the forwarder. -func (f *forwarder) close() { +func (f *Forwarder) Close() { close(f.cmds) } // log logs the current statistics. -func (f *forwarder) log() { - f.cmds <- func(f *forwarder) { +func (f *Forwarder) Log() { + f.cmds <- func(f *Forwarder) { slog.Info("Forward statistics", "succeeded", f.succeeded, "failed", f.failed) @@ -92,7 +92,7 @@ func (f *forwarder) log() { // httpClient returns a cached HTTP client used for uploading // the advisories to the configured HTTP endpoint. -func (f *forwarder) httpClient() util.Client { +func (f *Forwarder) httpClient() util.Client { if f.client != nil { return f.client } @@ -135,7 +135,7 @@ func replaceExt(fname, nExt string) string { } // buildRequest creates an HTTP request suited to forward the given advisory. -func (f *forwarder) buildRequest( +func (f *Forwarder) buildRequest( filename, doc string, status validationStatus, sha256, sha512 string, @@ -187,7 +187,7 @@ func (f *forwarder) buildRequest( // storeFailedAdvisory stores an advisory in a special folder // in case the forwarding failed. -func (f *forwarder) storeFailedAdvisory(filename, doc, sha256, sha512 string) error { +func (f *Forwarder) storeFailedAdvisory(filename, doc, sha256, sha512 string) error { // Create special folder if it does not exist. dir := filepath.Join(f.cfg.Directory, failedForwardDir) if err := os.MkdirAll(dir, 0755); err != nil { @@ -213,7 +213,7 @@ func (f *forwarder) storeFailedAdvisory(filename, doc, sha256, sha512 string) er } // storeFailed is a logging wrapper around storeFailedAdvisory. -func (f *forwarder) storeFailed(filename, doc, sha256, sha512 string) { +func (f *Forwarder) storeFailed(filename, doc, sha256, sha512 string) { f.failed++ if err := f.storeFailedAdvisory(filename, doc, sha256, sha512); err != nil { slog.Error("Storing advisory failed forwarding failed", @@ -237,13 +237,13 @@ func limitedString(r io.Reader, max int) (string, error) { // forward sends a given document with filename, status and // checksums to the forwarder. This is async to the degree // till the configured queue size is filled. -func (f *forwarder) forward( +func (f *Forwarder) forward( filename, doc string, status validationStatus, sha256, sha512 string, ) { // Run this in the main loop of the forwarder. - f.cmds <- func(f *forwarder) { + f.cmds <- func(f *Forwarder) { req, err := f.buildRequest(filename, doc, status, sha256, sha512) if err != nil { slog.Error("building forward Request failed", diff --git a/cmd/csaf_downloader/forwarder_test.go b/cmd/csaf_downloader/forwarder_test.go index 907bbcea..abfed42f 100644 --- a/cmd/csaf_downloader/forwarder_test.go +++ b/cmd/csaf_downloader/forwarder_test.go @@ -6,7 +6,7 @@ // SPDX-FileCopyrightText: 2023 German Federal Office for Information Security (BSI) // Software-Engineering: 2023 Intevation GmbH -package main +package csaf_downloader import ( "bufio" @@ -23,7 +23,7 @@ import ( "strings" "testing" - "github.com/csaf-poc/csaf_distribution/v3/internal/options" + "github.com/csaf-poc/csaf_distribution/v3/pkg/options" "github.com/csaf-poc/csaf_distribution/v3/util" ) @@ -53,18 +53,18 @@ func TestForwarderLogStats(t *testing.T) { lg := slog.New(h) slog.SetDefault(lg) - cfg := &config{} - fw := newForwarder(cfg) + cfg := &Config{} + fw := NewForwarder(cfg) fw.failed = 11 fw.succeeded = 13 done := make(chan struct{}) go func() { defer close(done) - fw.run() + fw.Run() }() - fw.log() - fw.close() + fw.Log() + fw.Close() <-done type fwStats struct { @@ -95,14 +95,14 @@ func TestForwarderLogStats(t *testing.T) { } func TestForwarderHTTPClient(t *testing.T) { - cfg := &config{ + cfg := &Config{ ForwardInsecure: true, ForwardHeader: http.Header{ "User-Agent": []string{"curl/7.55.1"}, }, LogLevel: &options.LogLevel{Level: slog.LevelDebug}, } - fw := newForwarder(cfg) + fw := NewForwarder(cfg) if c1, c2 := fw.httpClient(), fw.httpClient(); c1 != c2 { t.Fatal("expected to return same client twice") } @@ -124,10 +124,10 @@ func TestForwarderReplaceExtension(t *testing.T) { func TestForwarderBuildRequest(t *testing.T) { // Good case ... - cfg := &config{ + cfg := &Config{ ForwardURL: "https://example.com", } - fw := newForwarder(cfg) + fw := NewForwarder(cfg) req, err := fw.buildRequest( "test.json", "{}", @@ -248,8 +248,8 @@ func TestStoreFailedAdvisory(t *testing.T) { } defer os.RemoveAll(dir) - cfg := &config{Directory: dir} - fw := newForwarder(cfg) + cfg := &Config{Directory: dir} + fw := NewForwarder(cfg) badDir := filepath.Join(dir, failedForwardDir) if err := os.WriteFile(badDir, []byte("test"), 0664); err != nil { @@ -301,8 +301,8 @@ func TestStoredFailed(t *testing.T) { lg := slog.New(h) slog.SetDefault(lg) - cfg := &config{Directory: dir} - fw := newForwarder(cfg) + cfg := &Config{Directory: dir} + fw := NewForwarder(cfg) // An empty filename should lead to an error. fw.storeFailed("", "{}", "256", "512") @@ -385,11 +385,11 @@ func TestForwarderForward(t *testing.T) { lg := slog.New(h) slog.SetDefault(lg) - cfg := &config{ + cfg := &Config{ ForwardURL: "http://example.com", Directory: dir, } - fw := newForwarder(cfg) + fw := NewForwarder(cfg) // Use the fact that http client is cached. fw.client = &fakeClient{} @@ -398,7 +398,7 @@ func TestForwarderForward(t *testing.T) { go func() { defer close(done) - fw.run() + fw.Run() }() // Iterate through states of http client. @@ -412,7 +412,7 @@ func TestForwarderForward(t *testing.T) { // Make buildRequest fail. wait := make(chan struct{}) - fw.cmds <- func(f *forwarder) { + fw.cmds <- func(f *Forwarder) { f.cfg.ForwardURL = "%" close(wait) } @@ -423,7 +423,7 @@ func TestForwarderForward(t *testing.T) { "256", "512") - fw.close() + fw.Close() <-done } diff --git a/cmd/csaf_downloader/main.go b/cmd/csaf_downloader/main/main.go similarity index 66% rename from cmd/csaf_downloader/main.go rename to cmd/csaf_downloader/main/main.go index cc284bbc..dced3ef2 100644 --- a/cmd/csaf_downloader/main.go +++ b/cmd/csaf_downloader/main/main.go @@ -15,15 +15,16 @@ import ( "os" "os/signal" - "github.com/csaf-poc/csaf_distribution/v3/internal/options" + "github.com/csaf-poc/csaf_distribution/v3/cmd/csaf_downloader" + "github.com/csaf-poc/csaf_distribution/v3/pkg/options" ) -func run(cfg *config, domains []string) error { - d, err := newDownloader(cfg) +func run(cfg *csaf_downloader.Config, domains []string) error { + d, err := csaf_downloader.NewDownloader(cfg) if err != nil { return err } - defer d.close() + defer d.Close() ctx, cancel := context.WithCancel(context.Background()) defer cancel() @@ -31,28 +32,28 @@ func run(cfg *config, domains []string) error { defer stop() if cfg.ForwardURL != "" { - f := newForwarder(cfg) - go f.run() + f := csaf_downloader.NewForwarder(cfg) + go f.Run() defer func() { - f.log() - f.close() + f.Log() + f.Close() }() - d.forwarder = f + d.Forwarder = f } // If the enumerate-only flag is set, enumerate found PMDs, // else use the normal load method if cfg.EnumeratePMDOnly { - return d.runEnumerate(domains) + return d.RunEnumerate(domains) } - return d.run(ctx, domains) + return d.Run(ctx, domains) } func main() { - domains, cfg, err := parseArgsConfig() + domains, cfg, err := csaf_downloader.ParseArgsConfig() options.ErrorCheck(err) - options.ErrorCheck(cfg.prepare()) + options.ErrorCheck(cfg.Prepare()) if len(domains) == 0 { slog.Warn("No domains given.") diff --git a/cmd/csaf_downloader/stats.go b/cmd/csaf_downloader/stats.go index d85feb3a..c0c30828 100644 --- a/cmd/csaf_downloader/stats.go +++ b/cmd/csaf_downloader/stats.go @@ -6,7 +6,7 @@ // SPDX-FileCopyrightText: 2023 German Federal Office for Information Security (BSI) // Software-Engineering: 2023 Intevation GmbH -package main +package csaf_downloader import "log/slog" diff --git a/cmd/csaf_downloader/stats_test.go b/cmd/csaf_downloader/stats_test.go index 033fd7bf..be11415d 100644 --- a/cmd/csaf_downloader/stats_test.go +++ b/cmd/csaf_downloader/stats_test.go @@ -6,7 +6,7 @@ // SPDX-FileCopyrightText: 2023 German Federal Office for Information Security (BSI) // Software-Engineering: 2023 Intevation GmbH -package main +package csaf_downloader import ( "bytes" diff --git a/cmd/csaf_uploader/config.go b/cmd/csaf_uploader/config.go index a83361c7..8ad569a2 100644 --- a/cmd/csaf_uploader/config.go +++ b/cmd/csaf_uploader/config.go @@ -19,7 +19,7 @@ import ( "golang.org/x/term" "github.com/csaf-poc/csaf_distribution/v3/internal/certs" - "github.com/csaf-poc/csaf_distribution/v3/internal/options" + "github.com/csaf-poc/csaf_distribution/v3/pkg/options" ) const ( diff --git a/cmd/csaf_uploader/main.go b/cmd/csaf_uploader/main.go index 20f89fdf..91efda05 100644 --- a/cmd/csaf_uploader/main.go +++ b/cmd/csaf_uploader/main.go @@ -9,7 +9,7 @@ // Implements a command line tool that uploads csaf documents to csaf_provider. package main -import "github.com/csaf-poc/csaf_distribution/v3/internal/options" +import "github.com/csaf-poc/csaf_distribution/v3/pkg/options" func main() { args, cfg, err := parseArgsConfig() diff --git a/csaf/advisories.go b/csaf/advisories.go index 6f07648b..282c0f25 100644 --- a/csaf/advisories.go +++ b/csaf/advisories.go @@ -19,6 +19,7 @@ import ( "strings" "time" + "github.com/csaf-poc/csaf_distribution/v3/pkg/errs" "github.com/csaf-poc/csaf_distribution/v3/util" ) @@ -267,14 +268,24 @@ func (afp *AdvisoryFileProcessor) processROLIE( labeledFeeds []Feed, fn func(TLPLabel, []AdvisoryFile) error, ) error { + var feedErrs []error for i := range labeledFeeds { feed := &labeledFeeds[i] if feed.URL == nil { continue } + + var label TLPLabel + if feed.TLPLabel != nil { + label = *feed.TLPLabel + } else { + label = "unknown" + } + up, err := url.Parse(string(*feed.URL)) if err != nil { log.Printf("Invalid URL %s in feed: %v.", *feed.URL, err) + feedErrs = append(feedErrs, errs.ErrCsafProviderIssue{Message: fmt.Sprintf("invalid TLP:%s feed URL %s: %v", label, *feed.URL, err)}) continue } feedURL := afp.base.ResolveReference(up) @@ -283,22 +294,37 @@ func (afp *AdvisoryFileProcessor) processROLIE( fb, err := util.BaseURL(feedURL) if err != nil { log.Printf("error: Invalid feed base URL '%s': %v\n", fb, err) + feedErrs = append(feedErrs, errs.ErrCsafProviderIssue{Message: fmt.Sprintf("invalid TLP:%s feed base URL %s: %v", label, fb, err)}) continue } feedBaseURL, err := url.Parse(fb) if err != nil { log.Printf("error: Cannot parse feed base URL '%s': %v\n", fb, err) + feedErrs = append(feedErrs, errs.ErrCsafProviderIssue{Message: fmt.Sprintf("cannot parse TLP:%s feed base URL %s: %v", label, fb, err)}) continue } res, err := afp.client.Get(feedURL.String()) if err != nil { log.Printf("error: Cannot get feed '%s'\n", err) + feedErrs = append(feedErrs, errs.ErrNetwork{Message: fmt.Sprintf("failed get for TLP:%s feed url %s: %v", label, feedURL.String(), err)}) continue } if res.StatusCode != http.StatusOK { log.Printf("error: Fetching %s failed. Status code %d (%s)", feedURL, res.StatusCode, res.Status) + switch { + case res.StatusCode == http.StatusUnauthorized: + feedErrs = append(feedErrs, errs.ErrInvalidCredentials{Message: fmt.Sprintf("invalid credentials for TLP:%s ROLIE feed at %s: %s", label, feedURL.String(), res.Status)}) + case res.StatusCode == http.StatusNotFound: + feedErrs = append(feedErrs, errs.ErrCsafProviderIssue{Message: fmt.Sprintf("could not find TLP:%s ROLIE feed at %s: %s", label, feedURL.String(), res.Status)}) + case res.StatusCode == http.StatusForbidden: + // user has insufficient permissions to access feed, no error + case res.StatusCode > 500: + feedErrs = append(feedErrs, fmt.Errorf("could not retrieve TLP:%s ROLIE feed at %s: %s %w", label, feedURL.String(), res.Status, errs.ErrRetryable)) // mark error as retryable + default: + feedErrs = append(feedErrs, fmt.Errorf("could not retrieve TLP:%s ROLIE feed at %s: %s", label, feedURL.String(), res.Status)) + } continue } rfeed, err := func() (*ROLIEFeed, error) { @@ -307,21 +333,22 @@ func (afp *AdvisoryFileProcessor) processROLIE( }() if err != nil { log.Printf("Loading ROLIE feed failed: %v.", err) + feedErrs = append(feedErrs, errs.ErrCsafProviderIssue{Message: fmt.Sprintf("TLP:%s ROLIE feed at %s is not valid JSON: %v", label, feedURL.String(), err)}) continue } var files []AdvisoryFile - resolve := func(u string) string { + resolve := func(u string) (string, error) { if u == "" { - return "" + return "", errs.ErrCsafProviderIssue{Message: fmt.Sprintf("empty url in TLP:%s ROLIE feed at %s to file", label, feedURL.String())} } p, err := url.Parse(u) if err != nil { log.Printf("error: Invalid URL '%s': %v", u, err) - return "" + return "", errs.ErrCsafProviderIssue{Message: fmt.Sprintf("invalid url in TLP:%s ROLIE feed at %s to file %s: %v", label, feedURL.String(), u, err)} } - return feedBaseURL.ResolveReference(p).String() + return feedBaseURL.ResolveReference(p).String(), nil } rfeed.Entries(func(entry *Entry) { @@ -335,26 +362,41 @@ func (afp *AdvisoryFileProcessor) processROLIE( var self, sha256, sha512, sign string + var csafLinkExists bool for i := range entry.Link { link := &entry.Link[i] lower := strings.ToLower(link.HRef) switch link.Rel { case "self": - self = resolve(link.HRef) + csafLinkExists = true + self, err = resolve(link.HRef) + if err != nil { + feedErrs = append(feedErrs, err) + return + } case "signature": - sign = resolve(link.HRef) + sign, err = resolve(link.HRef) + if err != nil { + feedErrs = append(feedErrs, err) + } case "hash": switch { case strings.HasSuffix(lower, ".sha256"): - sha256 = resolve(link.HRef) + sha256, err = resolve(link.HRef) + if err != nil { + feedErrs = append(feedErrs, err) + } case strings.HasSuffix(lower, ".sha512"): - sha512 = resolve(link.HRef) + sha512, err = resolve(link.HRef) + if err != nil { + feedErrs = append(feedErrs, err) + } } } } - if self == "" { - return + if !csafLinkExists { + feedErrs = append(feedErrs, errs.ErrCsafProviderIssue{Message: fmt.Sprintf("TLP:%s ROLIE feed at %s contains entry (ID '%s') without link to csaf document", label, feedURL.String(), entry.ID)}) } var file AdvisoryFile @@ -368,16 +410,12 @@ func (afp *AdvisoryFileProcessor) processROLIE( files = append(files, file) }) - var label TLPLabel - if feed.TLPLabel != nil { - label = *feed.TLPLabel - } else { - label = "unknown" - } - if err := fn(label, files); err != nil { - return err + feedErrs = append(feedErrs, err) } } + if len(feedErrs) > 0 { + return &errs.CompositeErrRolieFeed{Errs: feedErrs} + } return nil } diff --git a/go.mod b/go.mod index 5b33724c..bbf537e4 100644 --- a/go.mod +++ b/go.mod @@ -1,31 +1,37 @@ module github.com/csaf-poc/csaf_distribution/v3 -go 1.21 +go 1.22.5 require ( - github.com/BurntSushi/toml v1.3.2 + github.com/BurntSushi/toml v1.4.0 github.com/Intevation/gval v1.3.0 github.com/Intevation/jsonpath v0.2.1 - github.com/ProtonMail/gopenpgp/v2 v2.7.4 - github.com/PuerkitoBio/goquery v1.8.1 - github.com/gofrs/flock v0.8.1 - github.com/jessevdk/go-flags v1.5.0 + github.com/ProtonMail/gopenpgp/v2 v2.7.5 + github.com/PuerkitoBio/goquery v1.9.2 + github.com/gofrs/flock v0.12.1 + github.com/jessevdk/go-flags v1.6.1 github.com/mitchellh/go-homedir v1.1.0 github.com/santhosh-tekuri/jsonschema/v5 v5.3.1 - go.etcd.io/bbolt v1.3.8 - golang.org/x/crypto v0.14.0 - golang.org/x/term v0.13.0 - golang.org/x/time v0.3.0 + github.com/stretchr/testify v1.9.0 + go.etcd.io/bbolt v1.3.11 + golang.org/x/crypto v0.26.0 + golang.org/x/term v0.23.0 + golang.org/x/time v0.6.0 ) require ( - github.com/ProtonMail/go-crypto v0.0.0-20230923063757-afb1ddc0824c // indirect + github.com/ProtonMail/go-crypto v1.1.0-alpha.5-proton // indirect github.com/ProtonMail/go-mime v0.0.0-20230322103455-7d82a3887f2f // indirect github.com/andybalholm/cascadia v1.3.2 // indirect - github.com/cloudflare/circl v1.3.6 // indirect + github.com/cloudflare/circl v1.4.0 // indirect + github.com/davecgh/go-spew v1.1.1 // indirect + github.com/kr/text v0.2.0 // indirect github.com/pkg/errors v0.9.1 // indirect - github.com/shopspring/decimal v1.3.1 // indirect - golang.org/x/net v0.17.0 // indirect - golang.org/x/sys v0.14.0 // indirect - golang.org/x/text v0.13.0 // indirect + github.com/pmezard/go-difflib v1.0.0 // indirect + github.com/rogpeppe/go-internal v1.12.0 // indirect + github.com/shopspring/decimal v1.4.0 // indirect + golang.org/x/net v0.28.0 // indirect + golang.org/x/sys v0.24.0 // indirect + golang.org/x/text v0.17.0 // indirect + gopkg.in/yaml.v3 v3.0.1 // indirect ) diff --git a/go.sum b/go.sum index f81653dc..c4879e0a 100644 --- a/go.sum +++ b/go.sum @@ -1,77 +1,82 @@ -github.com/BurntSushi/toml v1.3.2 h1:o7IhLm0Msx3BaB+n3Ag7L8EVlByGnpq14C4YWiu/gL8= -github.com/BurntSushi/toml v1.3.2/go.mod h1:CxXYINrC8qIiEnFrOxCa7Jy5BFHlXnUU2pbicEuybxQ= +github.com/BurntSushi/toml v1.4.0 h1:kuoIxZQy2WRRk1pttg9asf+WVv6tWQuBNVmK8+nqPr0= +github.com/BurntSushi/toml v1.4.0/go.mod h1:ukJfTF/6rtPPRCnwkur4qwRxa8vTRFBF0uk2lLoLwho= github.com/Intevation/gval v1.3.0 h1:+Ze5sft5MmGbZrHj06NVUbcxCb67l9RaPTLMNr37mjw= github.com/Intevation/gval v1.3.0/go.mod h1:xmGyGpP5be12EL0P12h+dqiYG8qn2j3PJxIgkoOHO5o= github.com/Intevation/jsonpath v0.2.1 h1:rINNQJ0Pts5XTFEG+zamtdL7l9uuE1z0FBA+r55Sw+A= github.com/Intevation/jsonpath v0.2.1/go.mod h1:WnZ8weMmwAx/fAO3SutjYFU+v7DFreNYnibV7CiaYIw= github.com/ProtonMail/go-crypto v0.0.0-20230717121422-5aa5874ade95/go.mod h1:EjAoLdwvbIOoOQr3ihjnSoLZRtE8azugULFRteWMNc0= -github.com/ProtonMail/go-crypto v0.0.0-20230923063757-afb1ddc0824c h1:kMFnB0vCcX7IL/m9Y5LO+KQYv+t1CQOiFe6+SV2J7bE= -github.com/ProtonMail/go-crypto v0.0.0-20230923063757-afb1ddc0824c/go.mod h1:EjAoLdwvbIOoOQr3ihjnSoLZRtE8azugULFRteWMNc0= +github.com/ProtonMail/go-crypto v1.1.0-alpha.5-proton h1:KVBEgU3CJpmzLChnLiSuEyCuhGhcMt3eOST+7A+ckto= +github.com/ProtonMail/go-crypto v1.1.0-alpha.5-proton/go.mod h1:rA3QumHc/FZ8pAHreoekgiAbzpNsfQAosU5td4SnOrE= github.com/ProtonMail/go-mime v0.0.0-20230322103455-7d82a3887f2f h1:tCbYj7/299ekTTXpdwKYF8eBlsYsDVoggDAuAjoK66k= github.com/ProtonMail/go-mime v0.0.0-20230322103455-7d82a3887f2f/go.mod h1:gcr0kNtGBqin9zDW9GOHcVntrwnjrK+qdJ06mWYBybw= -github.com/ProtonMail/gopenpgp/v2 v2.7.4 h1:Vz/8+HViFFnf2A6XX8JOvZMrA6F5puwNvvF21O1mRlo= -github.com/ProtonMail/gopenpgp/v2 v2.7.4/go.mod h1:IhkNEDaxec6NyzSI0PlxapinnwPVIESk8/76da3Ct3g= -github.com/PuerkitoBio/goquery v1.8.1 h1:uQxhNlArOIdbrH1tr0UXwdVFgDcZDrZVdcpygAcwmWM= -github.com/PuerkitoBio/goquery v1.8.1/go.mod h1:Q8ICL1kNUJ2sXGoAhPGUdYDJvgQgHzJsnnd3H7Ho5jQ= -github.com/andybalholm/cascadia v1.3.1/go.mod h1:R4bJ1UQfqADjvDa4P6HZHLh/3OxWWEqc0Sk8XGwHqvA= +github.com/ProtonMail/gopenpgp/v2 v2.7.5 h1:STOY3vgES59gNgoOt2w0nyHBjKViB/qSg7NjbQWPJkA= +github.com/ProtonMail/gopenpgp/v2 v2.7.5/go.mod h1:IhkNEDaxec6NyzSI0PlxapinnwPVIESk8/76da3Ct3g= +github.com/PuerkitoBio/goquery v1.9.2 h1:4/wZksC3KgkQw7SQgkKotmKljk0M6V8TUvA8Wb4yPeE= +github.com/PuerkitoBio/goquery v1.9.2/go.mod h1:GHPCaP0ODyyxqcNoFGYlAprUFH81NuRPd0GX3Zu2Mvk= github.com/andybalholm/cascadia v1.3.2 h1:3Xi6Dw5lHF15JtdcmAHD3i1+T8plmv7BQ/nsViSLyss= github.com/andybalholm/cascadia v1.3.2/go.mod h1:7gtRlve5FxPPgIgX36uWBX58OdBsSS6lUvCFb+h7KvU= github.com/bwesterb/go-ristretto v1.2.3/go.mod h1:fUIoIZaG73pV5biE2Blr2xEzDoMj7NFEuV9ekS419A0= github.com/cloudflare/circl v1.3.3/go.mod h1:5XYMA4rFBvNIrhs50XuiBJ15vF2pZn4nnUKZrLbUZFA= -github.com/cloudflare/circl v1.3.6 h1:/xbKIqSHbZXHwkhbrhrt2YOHIwYJlXH94E3tI/gDlUg= -github.com/cloudflare/circl v1.3.6/go.mod h1:5XYMA4rFBvNIrhs50XuiBJ15vF2pZn4nnUKZrLbUZFA= +github.com/cloudflare/circl v1.4.0 h1:BV7h5MgrktNzytKmWjpOtdYrf0lkkbF8YMlBGPhJQrY= +github.com/cloudflare/circl v1.4.0/go.mod h1:PDRU+oXvdD7KCtgKxW95M5Z8BpSCJXQORiZFnBQS5QU= +github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E= github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= -github.com/gofrs/flock v0.8.1 h1:+gYjHKf32LDeiEEFhQaotPbLuUXjY5ZqxKgXy7n59aw= -github.com/gofrs/flock v0.8.1/go.mod h1:F1TvTiK9OcQqauNUHlbJvyl9Qa1QvF/gOUDKA14jxHU= +github.com/gofrs/flock v0.12.1 h1:MTLVXXHf8ekldpJk3AKicLij9MdwOWkZ+a/jHHZby9E= +github.com/gofrs/flock v0.12.1/go.mod h1:9zxTsyu5xtJ9DK+1tFZyibEV7y3uwDxPPfbxeeHCoD0= github.com/google/go-cmp v0.5.9 h1:O2Tfq5qg4qc4AmwVlvv0oLiVAGB7enBSJ2x2DqQFi38= github.com/google/go-cmp v0.5.9/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= -github.com/jessevdk/go-flags v1.5.0 h1:1jKYvbxEjfUl0fmqTCOfonvskHHXMjBySTLW4y9LFvc= -github.com/jessevdk/go-flags v1.5.0/go.mod h1:Fw0T6WPc1dYxT4mKEZRfG5kJhaTDP9pj1c2EWnYs/m4= +github.com/jessevdk/go-flags v1.6.1 h1:Cvu5U8UGrLay1rZfv/zP7iLpSHGUZ/Ou68T0iX1bBK4= +github.com/jessevdk/go-flags v1.6.1/go.mod h1:Mk8T1hIAWpOiJiHa9rJASDK2UGWji0EuPGBnNLMooyc= +github.com/kr/pretty v0.3.1 h1:flRD4NNwYAUpkphVc1HcthR4KEIFJ65n8Mw5qdRn3LE= +github.com/kr/pretty v0.3.1/go.mod h1:hoEshYVHaxMs3cyo3Yncou5ZscifuDolrwPKZanG3xk= +github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= +github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE= github.com/mitchellh/go-homedir v1.1.0 h1:lukF9ziXFxDFPkA1vsr5zpc1XuPDn/wFntq5mG+4E0Y= github.com/mitchellh/go-homedir v1.1.0/go.mod h1:SfyaCUpYCn1Vlf4IUYiD9fPX4A5wJrkLzIz1N1q0pr0= github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4= github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= +github.com/rogpeppe/go-internal v1.12.0 h1:exVL4IDcn6na9z1rAb56Vxr+CgyK3nn3O+epU5NdKM8= +github.com/rogpeppe/go-internal v1.12.0/go.mod h1:E+RYuTGaKKdloAfM02xzb0FW3Paa99yedzYV+kq4uf4= github.com/santhosh-tekuri/jsonschema/v5 v5.3.1 h1:lZUw3E0/J3roVtGQ+SCrUrg3ON6NgVqpn3+iol9aGu4= github.com/santhosh-tekuri/jsonschema/v5 v5.3.1/go.mod h1:uToXkOrWAZ6/Oc07xWQrPOhJotwFIyu2bBVN41fcDUY= -github.com/shopspring/decimal v1.3.1 h1:2Usl1nmF/WZucqkFZhnfFYxxxu8LG21F6nPQBE5gKV8= github.com/shopspring/decimal v1.3.1/go.mod h1:DKyhrW/HYNuLGql+MJL6WCR6knT2jwCFRcu2hWCYk4o= +github.com/shopspring/decimal v1.4.0 h1:bxl37RwXBklmTi0C79JfXCEBD1cqqHt0bbgBAGFp81k= +github.com/shopspring/decimal v1.4.0/go.mod h1:gawqmDU56v4yIKSwfBSFip1HdCCXN8/+DMd9qYNcwME= github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= -github.com/stretchr/testify v1.8.1 h1:w7B6lhMri9wdJUVmEZPGGhZzrYTPvgJArz7wNPgYKsk= -github.com/stretchr/testify v1.8.1/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4= +github.com/stretchr/testify v1.9.0 h1:HtqpIVDClZ4nwg75+f6Lvsy/wHu+3BoSGCbBAcpTsTg= +github.com/stretchr/testify v1.9.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY= github.com/yuin/goldmark v1.4.13/go.mod h1:6yULJ656Px+3vBD8DxQVa3kxgyrAnzto9xy5taEt/CY= -go.etcd.io/bbolt v1.3.8 h1:xs88BrvEv273UsB79e0hcVrlUWmS0a8upikMFhSyAtA= -go.etcd.io/bbolt v1.3.8/go.mod h1:N9Mkw9X8x5fupy0IKsmuqVtoGDyxsaDlbk4Rd05IAQw= +go.etcd.io/bbolt v1.3.11 h1:yGEzV1wPz2yVCLsD8ZAiGHhHVlczyC9d1rP43/VCRJ0= +go.etcd.io/bbolt v1.3.11/go.mod h1:dksAq7YMXoljX0xu6VF5DMZGbhYYoLUalEiSySYAS4I= golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= golang.org/x/crypto v0.3.1-0.20221117191849-2c476679df9a/go.mod h1:hebNnKkNXi2UzZN1eVRvBB7co0a+JxK6XbPiWVs/3J4= golang.org/x/crypto v0.7.0/go.mod h1:pYwdfH91IfpZVANVyUOhSIPZaFoJGxTFbZhFTx+dXZU= -golang.org/x/crypto v0.14.0 h1:wBqGXzWJW6m1XrIKlAH0Hs1JJ7+9KBwnIO8v66Q9cHc= -golang.org/x/crypto v0.14.0/go.mod h1:MVFd36DqK4CsrnJYDkBA3VC4m2GkXAM0PvzMCn4JQf4= +golang.org/x/crypto v0.26.0 h1:RrRspgV4mU+YwB4FYnuBoKsUapNIL5cohGAmSH3azsw= +golang.org/x/crypto v0.26.0/go.mod h1:GY7jblb9wI+FOo5y8/S2oY4zWP07AkOJ4+jxCqdqn54= golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4/go.mod h1:jJ57K6gSWd91VN4djpZkiMVwK6gcyfeH4XE8wZrZaV4= golang.org/x/mod v0.8.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs= golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= -golang.org/x/net v0.0.0-20210916014120-12bc252f5db8/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= golang.org/x/net v0.0.0-20220722155237-a158d28d115b/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c= golang.org/x/net v0.2.0/go.mod h1:KqCZLdyyvdV855qA2rE3GC2aiw5xGR5TEjj8smXukLY= golang.org/x/net v0.6.0/go.mod h1:2Tu9+aMcznHK/AK1HMvgo6xiTLG5rD5rZLDS+rp2Bjs= -golang.org/x/net v0.7.0/go.mod h1:2Tu9+aMcznHK/AK1HMvgo6xiTLG5rD5rZLDS+rp2Bjs= golang.org/x/net v0.8.0/go.mod h1:QVkue5JL9kW//ek3r6jTKnTFis1tRmNAW2P1shuFdJc= golang.org/x/net v0.9.0/go.mod h1:d48xBJpPfHeWQsugry2m+kC02ZBRGRgulfHnEXEuWns= -golang.org/x/net v0.17.0 h1:pVaXccu2ozPjCXewfr1S7xza/zcXTity9cCdXQYSjIM= -golang.org/x/net v0.17.0/go.mod h1:NxSsAGuq816PNPmqtQdLE42eU2Fs7NoRIZrHJAlaCOE= +golang.org/x/net v0.28.0 h1:a9JDOJc5GMUJ0+UDqmLT86WiEy7iWyIhz8gz8E4e5hE= +golang.org/x/net v0.28.0/go.mod h1:yqtgsTWOOnlGLG9GFRrK3++bGOUEkNBoHZc8MEDWPNg= golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.1.0/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.8.0 h1:3NFvSEYkUoMifnESzZl15y791HH1qU2xm6eCJU5ZPXQ= +golang.org/x/sync v0.8.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk= golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20210320140829-1e4c9ba3b0c4/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20210423082822-04245dca01da/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220722155257-8c9f86f7a55f/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= @@ -80,35 +85,35 @@ golang.org/x/sys v0.3.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.5.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.7.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.14.0 h1:Vz7Qs629MkJkGyHxUlRHizWJRG2j8fbQKjELVSNhy7Q= -golang.org/x/sys v0.14.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= +golang.org/x/sys v0.24.0 h1:Twjiwq9dn6R1fQcyiK+wQyHWfaz/BJB+YIpzU/Cv3Xg= +golang.org/x/sys v0.24.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= golang.org/x/term v0.2.0/go.mod h1:TVmDHMZPmdnySmBfhjOoOdhjzdE1h4u1VwSiw2l1Nuc= golang.org/x/term v0.5.0/go.mod h1:jMB1sMXY+tzblOD4FWmEbocvup2/aLOaQEp7JmGp78k= golang.org/x/term v0.6.0/go.mod h1:m6U89DPEgQRMq3DNkDClhWw02AUbt2daBVO4cn4Hv9U= golang.org/x/term v0.7.0/go.mod h1:P32HKFT3hSsZrRxla30E9HqToFYAQPCMs/zFMBUFqPY= -golang.org/x/term v0.13.0 h1:bb+I9cTfFazGW51MZqBVmZy7+JEJMouUHTUSKVQLBek= -golang.org/x/term v0.13.0/go.mod h1:LTmsnFJwVN6bCy1rVCoS+qHT1HhALEFxKncY3WNNh4U= +golang.org/x/term v0.23.0 h1:F6D4vR+EHoL9/sWAWgAR1H2DcHr4PareCbAaCo1RpuU= +golang.org/x/term v0.23.0/go.mod h1:DgV24QBUrK6jhZXl+20l6UWznPlwAHm1Q1mGHtydmSk= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= -golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ= golang.org/x/text v0.4.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8= golang.org/x/text v0.7.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8= golang.org/x/text v0.8.0/go.mod h1:e1OnstbJyHTd6l/uOt8jFFHp6TRDWZR/bV3emEE/zU8= golang.org/x/text v0.9.0/go.mod h1:e1OnstbJyHTd6l/uOt8jFFHp6TRDWZR/bV3emEE/zU8= -golang.org/x/text v0.13.0 h1:ablQoSUd0tRdKxZewP80B+BaqeKJuVhuRxj/dkrun3k= -golang.org/x/text v0.13.0/go.mod h1:TvPlkZtksWOMsz7fbANvkp4WM8x/WCo/om8BMLbz+aE= -golang.org/x/time v0.3.0 h1:rg5rLMjNzMS1RkNLzCG38eapWhnYLFYXDXj2gOlr8j4= -golang.org/x/time v0.3.0/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= +golang.org/x/text v0.17.0 h1:XtiM5bkSOt+ewxlOE/aE/AKEHibwj/6gvWMl9Rsh0Qc= +golang.org/x/text v0.17.0/go.mod h1:BuEKDfySbSR4drPmRPG/7iBdf8hvFMuRexcpahXilzY= +golang.org/x/time v0.6.0 h1:eTDhh4ZXt5Qf0augr54TN6suAUudPcawVZeIAPU7D4U= +golang.org/x/time v0.6.0/go.mod h1:3BpzKBy/shNhVucY/MWOyx10tF3SFh9QdLuxbVysPQM= golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.1.12/go.mod h1:hNGJHUnrk76NpqgfD5Aqm5Crs+Hm0VOH/i9J2+nxYbc= golang.org/x/tools v0.6.0/go.mod h1:Xwgl3UAJ/d3gWutnCtw505GrjyAbvKui8lOU390QaIU= golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= -gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405 h1:yhCVgyC4o1eVCa2tZl7eS0r+SDo693bJlVdllGtEeKM= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk= +gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EVd6muEfDQjcINNoR0C8j2r3qZ4Q= gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= diff --git a/pkg/errs/errors.go b/pkg/errs/errors.go new file mode 100644 index 00000000..fa59c270 --- /dev/null +++ b/pkg/errs/errors.go @@ -0,0 +1,112 @@ +// This file is Free Software under the MIT License +// without warranty, see README.md and LICENSES/MIT.txt for details. +// +// SPDX-License-Identifier: MIT + +package errs + +import ( + "errors" + "strings" +) + +// ErrNetwork indicates a network level error +type ErrNetwork struct { + Message string +} + +func (e ErrNetwork) Error() string { + return e.Message +} + +// ErrInvalidCsaf notifies about an invalid csaf document (can only be fixed by the CSAF Source/Provider) +type ErrInvalidCsaf struct { + Message string +} + +func (e ErrInvalidCsaf) Error() string { + return e.Message +} + +// ErrCsafProviderIssue is an error which is not related directly the contents of a csaf document and can be only fixed by the CSAF Source/Provider +type ErrCsafProviderIssue struct { + Message string +} + +func (e ErrCsafProviderIssue) Error() string { + return e.Message +} + +type ErrInvalidCredentials struct { + Message string +} + +func (e ErrInvalidCredentials) Error() string { + return e.Message +} + +var ErrRetryable = errors.New("(retryable error)") + +// CompositeErrRolieFeed holds an array of errors which encountered during processing rolie feeds +type CompositeErrRolieFeed struct { + Errs []error +} + +func (e *CompositeErrRolieFeed) Error() string { + if len(e.Errs) == 0 { + return "empty CompositeErrRolieFeed" + } + + messages := make([]string, 0, len(e.Errs)) + for _, e := range e.Errs { + messages = append(messages, e.Error()) + } + return strings.Join(messages, "\n") +} + +func (e *CompositeErrRolieFeed) Unwrap() []error { + return e.Errs +} + +// CompositeErrCsafDownload holds an array of errors which encountered during the actual csaf download +type CompositeErrCsafDownload struct { + Errs []error +} + +func (e *CompositeErrCsafDownload) Error() string { + if len(e.Errs) == 0 { + return "empty CompositeErrCsafDownload" + } + + messages := make([]string, 0, len(e.Errs)) + for _, e := range e.Errs { + messages = append(messages, e.Error()) + } + return strings.Join(messages, "\n") +} + +func (e *CompositeErrCsafDownload) Unwrap() []error { + return e.Errs +} + +// FlattenError flattens out all composite errors (note: discards the errors wrapped around [CompositeErrRolieFeed] or [CompositeErrCsafDownload]) +// The assumed structure is CompositeErrRolieFeed{Errs: []error{...,CompositeErrCsafDownload,...,CompositeErrCsafDownload,...}}. +func FlattenError(err error) (flattenedErrors []error) { + var rolieErrs *CompositeErrRolieFeed + if errors.As(err, &rolieErrs) { + for _, rolieErr := range rolieErrs.Unwrap() { + var csafDlErrs *CompositeErrCsafDownload + if errors.As(rolieErr, &csafDlErrs) { + for _, csafDlErr := range csafDlErrs.Unwrap() { + flattenedErrors = append(flattenedErrors, csafDlErr) + } + } else { + flattenedErrors = append(flattenedErrors, rolieErr) + } + } + } else { + flattenedErrors = []error{err} + } + + return flattenedErrors +} diff --git a/pkg/errs/errors_test.go b/pkg/errs/errors_test.go new file mode 100644 index 00000000..1bb03b0b --- /dev/null +++ b/pkg/errs/errors_test.go @@ -0,0 +1,54 @@ +// This file is Free Software under the MIT License +// without warranty, see README.md and LICENSES/MIT.txt for details. +// +// SPDX-License-Identifier: MIT + +package errs + +import ( + "errors" + "fmt" + "slices" + "testing" + + "github.com/stretchr/testify/assert" +) + +func TestFlattenError(t *testing.T) { + + t.Run("flatten (only) composite errors defined in this package", func(t *testing.T) { + csafDownloadErrsFlat := []error{ + fmt.Errorf("error containing several errors 1: %w 2: %w", errors.New("nested err 1"), errors.New("nested err 2")), errors.New("nested err 2"), + errors.Join(errors.New("nested err in join 1"), errors.New("nested err in join 2")), + errors.New("single error 1"), + errors.New("single error 2"), + } + + compositeErrCsafDownload := &CompositeErrCsafDownload{Errs: csafDownloadErrsFlat} + + singleRolieFeedErrs := []error{ + errors.New("single error rolie feed 1"), + errors.New("single error rolie feed 2"), + } + + rolieFeedCompositeErr := CompositeErrRolieFeed{ + Errs: append( + singleRolieFeedErrs, + fmt.Errorf("issues during downloader of rolie: %w", compositeErrCsafDownload), + compositeErrCsafDownload, + ), + } + wantFlattenedErrors := slices.Concat(singleRolieFeedErrs, csafDownloadErrsFlat, csafDownloadErrsFlat) + + gotFlattenedErrors := FlattenError(fmt.Errorf("wrap rolie feed composite err: %w", &rolieFeedCompositeErr)) + + assert.ElementsMatch(t, wantFlattenedErrors, gotFlattenedErrors) + }) + + t.Run("single error is returned as is", func(t *testing.T) { + err := errors.Join(errors.New("nested err in join 1"), errors.New("nested err in join 2")) + wantFlattenedErrors := []error{err} + gotFlattenedErrors := FlattenError(err) + assert.ElementsMatch(t, wantFlattenedErrors, gotFlattenedErrors) + }) +} diff --git a/internal/models/models.go b/pkg/models/models.go similarity index 100% rename from internal/models/models.go rename to pkg/models/models.go diff --git a/internal/models/models_test.go b/pkg/models/models_test.go similarity index 100% rename from internal/models/models_test.go rename to pkg/models/models_test.go diff --git a/internal/options/data/config.toml b/pkg/options/data/config.toml similarity index 100% rename from internal/options/data/config.toml rename to pkg/options/data/config.toml diff --git a/internal/options/data/config_plus.toml b/pkg/options/data/config_plus.toml similarity index 100% rename from internal/options/data/config_plus.toml rename to pkg/options/data/config_plus.toml diff --git a/internal/options/data/empty.toml b/pkg/options/data/empty.toml similarity index 100% rename from internal/options/data/empty.toml rename to pkg/options/data/empty.toml diff --git a/internal/options/log.go b/pkg/options/log.go similarity index 100% rename from internal/options/log.go rename to pkg/options/log.go diff --git a/internal/options/log_test.go b/pkg/options/log_test.go similarity index 100% rename from internal/options/log_test.go rename to pkg/options/log_test.go diff --git a/internal/options/options.go b/pkg/options/options.go similarity index 100% rename from internal/options/options.go rename to pkg/options/options.go diff --git a/internal/options/options_test.go b/pkg/options/options_test.go similarity index 100% rename from internal/options/options_test.go rename to pkg/options/options_test.go diff --git a/version.go b/version.go new file mode 100644 index 00000000..631b0272 --- /dev/null +++ b/version.go @@ -0,0 +1,6 @@ +package main + +// THIS IS AN AUTOGENERATED FILE. DO NOT TOUCH! + +var version = "3.2.4-dev1" +