Add upstream changes (#40)

Monthly update of bringing the upstream changes to our fork. There are
no significant changes to the csaf downloader.

cmd/csaf_aggregator/config.go

@@ -301,6 +301,11 @@ func (c *config) httpClient(p *provider) util.Client {
 			Client: client,
 			Header: c.ExtraHeader,
 		}
+	default:
+		client = &util.HeaderClient{
+			Client: client,
+			Header: http.Header{},
+		}
 	}
 
 	if c.Verbose {

cmd/csaf_checker/processor.go

@@ -435,11 +435,9 @@ func (p *processor) fullClient() util.Client {
 	client := util.Client(&hClient)
 
 	// Add extra headers.
-	if len(p.cfg.ExtraHeader) > 0 {
-		client = &util.HeaderClient{
-			Client: client,
-			Header: p.cfg.ExtraHeader,
-		}
+	client = &util.HeaderClient{
+		Client: client,
+		Header: p.cfg.ExtraHeader,
 	}
 
 	// Add optional URL logging.

cmd/csaf_downloader/downloader.go

@@ -130,11 +130,9 @@ func (d *Downloader) httpClient() util.Client {
 	client := util.Client(&hClient)
 
 	// Add extra headers.
-	if len(d.cfg.ExtraHeader) > 0 {
-		client = &util.HeaderClient{
-			Client: client,
-			Header: d.cfg.ExtraHeader,
-		}
+	client = &util.HeaderClient{
+		Client: client,
+		Header: d.cfg.ExtraHeader,
 	}
 
 	// Add optional URL logging.

cmd/csaf_downloader/forwarder.go

@@ -111,11 +111,9 @@ func (f *Forwarder) httpClient() util.Client {
 	client := util.Client(&hClient)
 
 	// Add extra headers.
-	if len(f.cfg.ForwardHeader) > 0 {
-		client = &util.HeaderClient{
-			Client: client,
-			Header: f.cfg.ForwardHeader,
-		}
+	client = &util.HeaderClient{
+		Client: client,
+		Header: f.cfg.ForwardHeader,
 	}
 
 	// Add optional URL logging.

docs/csaf_aggregator.md

@@ -16,6 +16,7 @@ Help Options:
 ```
 
 If no config file is explictly given the follwing places are searched for a config file:
+
 ```
 ~/.config/csaf/aggregator.toml
 ~/.csaf_aggregator.toml
@@ -25,6 +26,7 @@ csaf_aggregator.toml
 with `~` expanding to `$HOME` on unixoid systems and `%HOMEPATH` on Windows systems.
 
 Usage example for a single run, to test if the config is good:
+
 ```bash
 ./csaf_aggregator -c docs/examples/aggregator.toml
 ```
@@ -62,7 +64,6 @@ SHELL=/bin/bash
 30 0-23 * * * $HOME/bin/csaf_aggregator --config /etc/csaf_aggregator.toml --interim >> /var/log/csaf_aggregator/interim.log 2>&1
 ```
 
-
 #### serve via web server
 
 Serve the paths where the aggregator writes its `html/` output
@@ -78,7 +79,6 @@ a template. For the aggregator the difference is that you can leave out
 the cgi-bin part, potentially commend out the TLS client parts and
 adjust the `root` path accordingly.
 
-
 ### config options
 
 The config file is written in [TOML](https://toml.io/en/v1.0.0).
@@ -118,10 +118,12 @@ Next we have two TOML _tables_:
 aggregator            // basic infos for the aggregator object
 remote_validator      // config for optional remote validation checker
 ```
+
 [See the provider config](csaf_provider.md#provider-options) about
 how to configure `remote_validator`.
 
 At last there is the TOML _array of tables_:
+
 ```
 providers             // each entry to be mirrored or listed
 ```
@@ -148,6 +150,9 @@ header
 
 Where valid `name` and `domain` settings are required.
 
+If no user agent is specified with `header = "user-agent:custom-agent/1.0"`
+then the default agent in the form of `csaf_distribution/VERSION` is sent.
+
 If you want an entry to be listed instead of mirrored
 in a `aggregator.category == "aggregator"` instance,
 set `category` to `lister` in the entry.
@@ -165,15 +170,16 @@ To offer an easy way of assorting CSAF documents by criteria like
 document category, languages or values of the branch category within
 the product tree, ROLIE category values can be configured in `categories`.
 This can either
-be done using an array of strings taken literally or, by prepending `"expr:"`. 
-The latter is evaluated as JSONPath and the result will be added into the 
+be done using an array of strings taken literally or, by prepending `"expr:"`.
+The latter is evaluated as JSONPath and the result will be added into the
 categories document. For a more detailed explanation and examples,
 [refer to the provider config](csaf_provider.md#provider-options).
 
-
 #### Example config file
+
 <!-- MARKDOWN-AUTO-DOCS:START (CODE:src=../docs/examples/aggregator.toml) -->
 <!-- The below code snippet is automatically added from ../docs/examples/aggregator.toml -->
+
 ```toml
 workers = 2
 folder = "/var/csaf_aggregator"
@@ -233,8 +239,8 @@ insecure = true
   category = "lister"
 # ignore_pattern = [".*white.*", ".*red.*"]
 ```
-<!-- MARKDOWN-AUTO-DOCS:END -->
 
+<!-- MARKDOWN-AUTO-DOCS:END -->
 
 #### Publish others' advisories
 

docs/csaf_checker.md

@@ -30,9 +30,12 @@ Help Options:
 
 Will check all given _domains_, by trying each as a CSAF provider.
 
+If no user agent is specified with `--header=user-agent:custom-agent/1.0` then the default agent in the form of `csaf_distribution/VERSION` is sent.
+
 If a _domain_ starts with `https://` it is instead considered a direct URL to the `provider-metadata.json` and checking proceeds from there.
 
 If no config file is explictly given the follwing places are searched for a config file:
+
 ```
 ~/.config/csaf/checker.toml
 ~/.csaf_checker.toml
@@ -41,6 +44,7 @@ csaf_checker.toml
 
 with `~` expanding to `$HOME` on unixoid systems and `%HOMEPATH` on Windows systems.
 Supported options in config files:
+
 ```
 output              = ""
 format              = "json"
@@ -58,9 +62,10 @@ validator_preset    = ["mandatory"]
 ```
 
 Usage example:
-` ./csaf_checker example.com -f html --rate=5.3 -H apikey:SECRET -o check-results.html`
+`./csaf_checker example.com -f html --rate=5.3 -H apikey:SECRET -o check-results.html`
 
 Each performed check has a return type of either 0,1 or 2:
+
 ```
 type 0: success
 type 1: warning
@@ -70,16 +75,16 @@ type 2: error
 The checker result is a success if no checks resulted in type 2, and a failure otherwise.
 
 The option `timerange` allows to only check advisories from a given time
-interval. It can only be given once.  See the
+interval. It can only be given once. See the
 [downloader documentation](csaf_downloader.md#timerange-option) for details.
 
-
 You can ignore certain advisories while checking by specifying a list
 of regular expressions[^1] to match their URLs by using the `ignorepattern`
 option.
 E.g. `-i='.*white.*' -i='*.red.*'` will ignore files which URLs contain
 the sub strings **white** or **red**.
 In the config file this has to be noted as:
+
 ```
 ignorepattern = [".*white.*", ".*red.*"]
 ```
@@ -88,7 +93,7 @@ ignorepattern = [".*white.*", ".*red.*"]
 
 The `role` given in the `provider-metadata.json` is not
 yet considered to change the overall result,
-see https://github.com/csaf-poc/csaf_distribution/issues/221 .
+see <https://github.com/csaf-poc/csaf_distribution/issues/221> .
 
 If a provider hosts one or more advisories with a TLP level of AMBER or RED, then these advisories must be access protected.
 To check these advisories, authorization can be given via custom headers or certificates.

docs/csaf_downloader.md

@@ -1,4 +1,5 @@
 ## csaf_downloader
+
 A tool to download CSAF documents from CSAF providers.
 
 ### Usage
@@ -39,6 +40,8 @@ Help Options:
 
 Will download all CSAF documents for the given _domains_, by trying each as a CSAF provider.
 
+If no user agent is specified with `--header=user-agent:custom-agent/1.0` then the default agent in the form of `csaf_distribution/VERSION` is sent.
+
 If a _domain_ starts with `https://` it is instead considered a direct URL to the `provider-metadata.json` and downloading procedes from there.
 
 Increasing the number of workers opens more connections to the web servers
@@ -47,6 +50,7 @@ However, since this also increases the load on the servers, their administrators
 have taken countermeasures to limit this.
 
 If no config file is explictly given the follwing places are searched for a config file:
+
 ```
 ~/.config/csaf/downloader.toml
 ~/.csaf_downloader.toml
@@ -56,6 +60,7 @@ csaf_downloader.toml
 with `~` expanding to `$HOME` on unixoid systems and `%HOMEPATH` on Windows systems.
 
 Supported options in config files:
+
 ```
 # directory         # not set by default
 insecure            = false
@@ -90,6 +95,7 @@ option.
 E.g. `-i='.*white.*' -i='*.red.*'` will ignore files which URLs contain
 the sub strings **white** or **red**.
 In the config file this has to be noted as:
+
 ```
 ignorepattern = [".*white.*", ".*red.*"]
 ```
@@ -106,16 +112,18 @@ into a given intervall. There are three possible notations:
    and 'y' for years are recognized. In these cases only integer
    values are accepted without any fractions.
    Some examples:
+
    - `"3h"` means downloading the advisories that have changed in the last three hours.
-   - `"30m"`  .. changed within the last thirty minutes.
+   - `"30m"` .. changed within the last thirty minutes.
    - `"3M2m"` .. changed within the last three months and two minutes.
-   - `"2y"`   .. changed within the last two years.
+   - `"2y"` .. changed within the last two years.
 
 2. Absolute. If the given string is an RFC 3339 date timestamp
    the time interval between this date and now is used.
    E.g. `"2006-01-02"` means that all files between 2006 January 2nd and now going to being
    downloaded.
    Accepted patterns are:
+
    - `"2006-01-02T15:04:05Z"`
    - `"2006-01-02T15:04:05+07:00"`
    - `"2006-01-02T15:04:05-07:00"`
@@ -134,6 +142,7 @@ into a given intervall. There are three possible notations:
 All interval boundaries are inclusive.
 
 #### Forwarding
+
 The downloader is able to forward downloaded advisories and their checksums,
 OpenPGP signatures and validation results to an HTTP endpoint.  
 The details of the implemented API are described [here](https://github.com/mfd2007/csaf_upload_interface).  

docs/scripts/Readme.md

@@ -12,10 +12,10 @@ and configures nginx for serving TLS connections.
 
 As creating the folders needs to authenticate with the csaf_provider, the configurations of TLS server and Client certificate authentication should be set. So it is recommended to call the scripts in this order: `TLSConfigsForITest.sh`, `TLSClientConfigsForITest.sh`, `setupProviderForITest.sh`
 
-Calling example (as root):
+Calling example (as user with sudo privileges):
 ``` bash
     curl --fail -O https://raw.githubusercontent.com/csaf-poc/csaf_distribution/main/docs/scripts/prepareUbuntuInstanceForITests.sh
-    bash prepareUbuntuInstanceForITests.sh
+    sudo bash prepareUbuntuInstanceForITests.sh
 
     git clone https://github.com/csaf-poc/csaf_distribution.git # --branch <name>
     pushd csaf_distribution/docs/scripts/

docs/scripts/setupValidationService.sh

@@ -21,7 +21,7 @@ echo '
 remote_validator= { "url" = "http://localhost:8082", "presets" = ["mandatory"], "cache" = "/var/lib/csaf/validations.db" }
 ' | sudo tee --append /etc/csaf/config.toml
 
-npm install pm2 -g
+sudo npm install pm2 -g
 
 pushd ~
 git clone https://github.com/secvisogram/csaf-validator-service.git

util/client.go

@@ -61,6 +61,11 @@ func (hc *HeaderClient) Do(req *http.Request) (*http.Response, error) {
 			req.Header.Add(key, v)
 		}
 	}
+
+	// Use default user agent if none is set
+	if userAgent := hc.Header.Get("User-Agent"); userAgent == "" {
+		req.Header.Add("User-Agent", "csaf_distribution/"+SemVersion)
+	}
 	return hc.Client.Do(req)
 }