From c946fa8407d872719a0ffa2ef09c6a3759eaf4c7 Mon Sep 17 00:00:00 2001 From: Lee Read Date: Fri, 16 Aug 2024 01:06:04 -0400 Subject: [PATCH] Encourage nvd api key usage (#99) * Encourage nvd api key usage Clj-watson now exits with an error when running without an nvd api key. A stubborn user can opt to proceed without an nvd api key by specifying `--run-without-nvd-api-key` (or `:run-without-nvd-api-key true`). In this case clj-watson will emit a warning to remind the user that they really should be using an nvd api key. Update README on this topic. Also tweaked README here and there with some edits. Gave GitHub Markdown alerts a whirl: https://docs.github.com/en/get-started/writing-on-github/getting-started-with-writing-and-formatting-on-github/basic-writing-and-formatting-syntax#alerts Closes #67 * Review feedback: `clj-watson` for consistency * review feedback: typo * review feedback: doc api key as system property Also added some cautions on keeping the api key secret --- README.md | 142 ++++++++++++------ deps.edn | 1 + src/clj_watson/cli_spec.clj | 30 +++- .../controller/dependency_check/scanner.clj | 40 +++-- src/clj_watson/entrypoint.clj | 5 +- 5 files changed, 155 insertions(+), 63 deletions(-) diff --git a/README.md b/README.md index 163447e..79b24cf 100644 --- a/README.md +++ b/README.md @@ -1,9 +1,9 @@ -# clj-watson +# `clj-watson` A Clojure tool for vulnerability checking. -`clj-watson` is a software composition analysis (SCA) tool, that scans -dependencies in a Clojure `deps.edn` file looking for vulnerable direct and +`clj-watson` is a software composition analysis (SCA) tool that scans +dependencies specified in a Clojure `deps.edn` file looking for vulnerable direct and transitive dependencies, and builds a report with all the information needed to help you understand how the vulnerabilities manifest in your software. @@ -16,7 +16,7 @@ and can check against both the ## Quick Start -`clj-watson` can be added as an alias either on a per-project basis in the +`clj-watson` can be added as an alias either on a per-project basis in your project's `deps.edn` file or in your user `deps.edn` file (either `~/.clojure/deps.edn` or `~/.config/clojure/deps.edn`): @@ -25,27 +25,31 @@ project's `deps.edn` file or in your user `deps.edn` file :clj-watson {:replace-deps {io.github.clj-holmes/clj-watson {:git/tag "v5.1.3" :git/sha "5812615"}} - :main-opts ["-m" "clj-watson.cli" "scan"]} + :main-opts ["-m" "clj-watson.cli"]} ``` +> [!IMPORTANT] +> You'll need to first [setup your NVD API key](#nist-nvd-api). + Then you can run it with: ```bash -clojure -M:clj-watson -p deps.edn +clojure -M:clj-watson scan -p deps.edn ``` -The first time it runs, it will download the vulnerability database, which -can take a few minutes. Subsequent runs will be much faster. +The first time it runs, it will download the entire vulnerability database, which +can take several minutes. Subsequent runs will be much faster. -> Note: the database is stored in the `/tmp/db/` folder (on macOS/Linux) - in case you ever need to delete that folder, if it looks like the database is corrupted. +> [!NOTE] +> The database is stored in the `/tmp/db/` folder (on macOS/Linux) - in case you ever need to delete that folder, if it looks like the database is corrupted. -It can also be installed as a Clojure CLI tool: +`clj-watson` can also be installed as a Clojure CLI tool: ```bash clojure -Ttools install-latest :lib io.github.clj-holmes/clj-watson :as clj-watson ``` -Then run it with: +Then can be run via: ```bash clojure -Tclj-watson scan :deps-edn-path '"deps.edn"' :output '"stdout"' @@ -65,50 +69,93 @@ clojure -Tclj-watson scan :p deps.edn `:output` can be omitted because it defaults to `stdout`, and `:deps-edn-path` can be shortened to `:p` (matching the `-p` short form of `--deps-edn-path`). -> Note: `:aliases` (or `:a`) should be specified as a vector of keywords (or symbols), e.g., `:a '[:foo :bar]`, whereas it would be specified multiple times (as strings) in the regular CLI, `-a foo -a bar`. +> [!NOTE] +> `:aliases` (or `:a`) should be specified as a vector of keywords (or symbols), e.g., `:a '[:foo :bar]`, whereas it would be specified multiple times (as strings) in the regular CLI, `-a foo -a bar`. # How it works ## Vulnerability database strategies -`clj-watson` supports two methods for vulnerabilities scan. +`clj-watson` supports two methods for vulnerabilities scans. ### DependencyCheck [DependencyCheck](https://github.com/jeremylong/DependencyCheck) is the most -widely used method among the Clojure/Java SCA tools. It downloads all -vulnerabilities from NVD and stores it in a database (located in the `/tmp/db/` -folder), composes a -[Common Platform Enumeration (CPE)](https://nvd.nist.gov/products/cpe) based on -the dependencies, scans all JARs in the classpath and matches vulnerabilities -using it. +widely used method among the Clojure/Java SCA tools. It: +1. Downloads a database of known vulnerabilities from [NIST NVD](https://nvd.nist.gov/), storing it locally under your `/tmp/db/` folder +3. Scans JARs from dependencies specified in your `deps.edn` +4. Composes a [Common Platform Enumeration (CPE)](https://nvd.nist.gov/products/cpe) based on your dependencies +5. Returns any matching vulnerabilities -* `clj-watson` v5.1.3 onward uses DependencyCheck 10.0.x and the new NIST NVD API. - * `clj-watson` v5.0.0..v5.1.2 used DependencyCheck 9.0.x which caused the NIST NVD API to be overwhelmed; please update to v5.1.3! -* `clj-watson` v4.x.x uses an earlier version of DependencyCheck and the old NVD data feeds, which have been deprecated. +`clj-watson` then reports these findings to you, optionally with [potential remediations](#remediation-suggestions). + +> [!IMPORTANT] +> We _always_ recommend using the latest version of `clj-watson`, but as a minimum upgrade to v5.1.3. +> All earlier versions of `clj-watson` are officially deprecated. +> Older versions of `clj-watson` use older problematic versions of DependencyCheck, which NIST is now blocking. #### NIST NVD API -As of version v5.0.0, `clj-watson` switched to -[`DependencyCheck` 9.0.x](https://github.com/jeremylong/DependencyCheck/tree/main?tab=readme-ov-file#900-upgrade-notice) -which switches from the earlier NVD data feeds to the new NIST NVD API. -**NIST are forcing everyone to upgrade to 10.0.3 or later so please use `clj-watson` v5.1.3 or later!** +> [!IMPORTANT] +> The [NIST NVD data feeds discourage access without API keys by heavily throttling anonymous requests](https://nvd.nist.gov/general/news/API-Key-Announcement). +> So, request one and use it. + +It is easy to [request an API key](https://github.com/jeremylong/DependencyCheck/tree/main?tab=readme-ov-file#nvd-api-key-highly-recommended). + +You can specify you key via: + +1. The `nvd.api.key` Java system property on the command line +2. Or, an `nvd.api.key` entry in your `clj-watson.properties` file + +> [!CAUTION] +> Keeping your nvd api key secret is your responsibility. +> This is not a hugely sensitive secret, but you don't want others to use your key. +> You do not want to check it into any public version control system. + +##### Via Java System Property on the Command Line + +Example usage: + +```shell +clojure -J-Dnvd.api.key= -M:clj-watson scan -p deps.edn +``` + +Or: + +```shell +clojure -J-Dnvd.api.key= -Tclj-watson scan :p deps.edn +``` + +Replace `` with your actual api key. -This new API heavily throttles anonymous requests, so it is -[highly recommended to get an API key](https://github.com/jeremylong/DependencyCheck/tree/main?tab=readme-ov-file#nvd-api-key-highly-recommended) -in order to use the API efficiently. +> [!CAUTION] +> You could specify this system property under `:jvm-opts` in your `deps.edn` under your `:clj-watson` alias, but be careful not to commit it to version control. -Read the [NIST NVD announcement](https://nvd.nist.gov/general/news/API-Key-Announcement) for more information. +##### Via the `clj-watson.properties` File -Once you have an API key, you can provide it to `clj-watson` via the `nvd.api.key` -property in the optional `clj-watson.properties` file, either on the classpath -you use to run `clj-watson` or via the `-w` / `--clj-watson-properties` -command-line option: +Specify your key in your `clj-watson.properties` file: ``` # clj-watson.properties file -nvd.api.key=...your key here... +nvd.api.key= ``` +Replace `` with your actual api key. + +`clj-watson` will pick up `clj-watson.properties` automatically if it is on the classpath, or you can specify it on the command line via the `-w` / `--clj-watson-properties` option: + + +```shell +clojure -M:clj-watson scan -p deps.edn --clj-watson-properties ./clj-watson.properties +``` + +Or: + +```shell +clojure -Tclj-watson scan :p deps.edn :clj-watson-properties ./clj-watson.properties +``` + +> [!CAUTION] +> Be careful not to commit your key to version control. ### GitHub Advisory Database [experimental] @@ -130,8 +177,8 @@ environment variable named `GITHUB_TOKEN` and `clj-watson` will be able to use i #### Allow Listing Known CVE's -Sometimes the transitive dependency tree is not under your control and it is -not always possible to override versions of dependencies that are vulnerable. +Sometimes, the transitive dependency tree is not under your control and it is +not always possible to override vulnerable dependencies. You can allow a CVE for a limited period by adding a `clj-watson-config.edn` configuration file to your classpath with the following structure: @@ -144,7 +191,7 @@ configuration file to your classpath with the following structure: > Note: this is for the GitHub Advisory Database strategy only. -## Remediation suggestion +## Remediation suggestions **The big difference between `clj-watson` and other tools!** @@ -185,10 +232,13 @@ the `--suggest-fix` or `-s` option when running `clj-watson`. # Installation +> [!IMPORTANT] +> You'll need to [setup your NVD API key](#nist-nvd-api). + `clj-watson` can be installed as a Clojure CLI tool, as shown above. While this is the easiest way to install the latest version and keep it up-to-date (using `clojure -Ttools install-latest`), it also means using the key/value -EDN-style options for the CLI tool which can be a bit unwieldy as present: +EDN-style options for the CLI tool, which can at first seem a bit unwieldy: ```bash clojure -Tclj-watson scan '{:output "stdout" :fail-on-result true :deps-edn-path "deps.edn" :suggest-fix true :aliases ["*"] :database-strategy "dependency-check"}' @@ -211,14 +261,14 @@ Or you can just add it to your `deps.edn` file as an alias: ```clojure {:deps {} :aliases - {:clj-watson {:extra-deps {io.github.clj-holmes/clj-watson {:git/tag "v5.1.3" :git/sha "5812615"}} - :main-opts ["-m" "clj-watson.cli" "scan"]}}} + {:clj-watson {:replace-deps {io.github.clj-holmes/clj-watson {:git/tag "v5.1.3" :git/sha "5812615"}} + :main-opts ["-m" "clj-watson.cli"]}}} ``` and invoke it with: ```bash -clojure -M:clj-watson -p deps.edn +clojure -M:clj-watson scan -p deps.edn ``` # CLI Options @@ -254,6 +304,9 @@ OPTIONS valid when database-strategy is dependency-check: -w, --clj-watson-properties Path of an additional, optional properties file Overrides values in dependency-check.properties If not specified classpath is searched for cljwatson.properties + --run-without-nvd-api-key Run without an nvd.api.key configured. + It will be slow and we cannot recommend it. + See docs for configuration. [false] ``` By default, when using the DEPENDENCY-CHECK strategy, `clj-watson` will load @@ -275,7 +328,10 @@ or the `-d` file. This can be useful to override just a few properties. The minimum needed to run `clj-watson` is to provide the path to a `deps.edn` file, but it is recommended that you also provide the `-s` option so -`clj-watson` will try to suggest a remediation for any vulnerabilities found. +`clj-watson` will try to suggest remediations for any vulnerabilities found. + +> [!IMPORTANT] +> You'll need to first [setup your NVD API key](#nist-nvd-api). ```bash clojure -M:clj-watson -p deps.edn diff --git a/deps.edn b/deps.edn index dfce1cb..488d1aa 100644 --- a/deps.edn +++ b/deps.edn @@ -1,4 +1,5 @@ {:deps {org.clojure/clojure {:mvn/version "1.11.4"} + org.clojure/tools.logging {:mvn/version "1.3.0"} org.babashka/cli {:mvn/version "0.8.60"} borkdude/edamame {:mvn/version "1.4.25"} cheshire/cheshire {:mvn/version "5.13.0"} diff --git a/src/clj_watson/cli_spec.clj b/src/clj_watson/cli_spec.clj index 7b73378..ca1a7ff 100644 --- a/src/clj_watson/cli_spec.clj +++ b/src/clj_watson/cli_spec.clj @@ -83,7 +83,14 @@ :validate validate-file-exists :desc (str "Path of an additional, optional properties file\n" "Overrides values in dependency-check.properties\n" - "If not specified classpath is searched for cljwatson.properties")}}) + "If not specified classpath is searched for cljwatson.properties")} + + :run-without-nvd-api-key + {:type :flag + :default false + :desc (str "Run without an nvd.api.key configured.\n" + "It will be slow and we cannot recommend it.\n" + "See docs for configuration.")}}) (defn- kw->str "Copied from bb cli" @@ -134,6 +141,16 @@ (map str/trimr) (str/join "\n")))) +(defn styled-long-opt [longopt {:keys [usage-help-style]}] + (if (= :clojure-tool usage-help-style) + longopt + (str "--" (kw->str longopt)))) + +(defn styled-alias [alias {:keys [usage-help-style]}] + (if (= :clojure-tool usage-help-style) + alias + (str "-" (kw->str alias)))) + (defn- opts->table "Based on bb cli opts->table." [{:keys [spec order opts]}] @@ -141,13 +158,9 @@ (mapv (fn [[long-opt {:keys [alias default default-desc ref desc extra-desc require]}]] (keep identity [(if alias - (if (= :clojure-tool usage-help-style) - (str alias ",") - (str "-" (kw->str alias) ",")) + (str (styled-alias alias opts) ",") "") - (if (= :clojure-tool usage-help-style) - (str long-opt " " ref) - (str "--" (kw->str long-opt) " " ref)) + (str (styled-long-opt long-opt opts) " " ref) (->> [(if-let [attribute (or (when require "*required*") default-desc (when (some? default) (str default)))] @@ -190,7 +203,7 @@ :groups [{:heading "OPTIONS:" :order [:deps-edn-path :output :aliases :database-strategy :suggest-fix :fail-on-result :help]} {:heading "OPTIONS valid when database-strategy is dependency-check:" - :order [:dependency-check-properties :clj-watson-properties]}]}))) + :order [:dependency-check-properties :clj-watson-properties :run-without-nvd-api-key]}]}))) (defn- error [text] (str "\u001B[31m* ERROR: " text "\u001B[0m")) @@ -246,6 +259,7 @@ :msg (format "Invalid command, the only valid command is scan, detected: %s" (str/join ", " args)) :spec spec-scan-args :opts opts}) + :else (cli/parse-opts orig-args {:spec spec-scan-args :error-fn usage-error :restrict true})))) diff --git a/src/clj_watson/controller/dependency_check/scanner.clj b/src/clj_watson/controller/dependency_check/scanner.clj index 7d16cad..624b8f8 100644 --- a/src/clj_watson/controller/dependency_check/scanner.clj +++ b/src/clj_watson/controller/dependency_check/scanner.clj @@ -1,7 +1,9 @@ (ns clj-watson.controller.dependency-check.scanner (:require + [clj-watson.cli-spec :as cli-spec] [clojure.java.io :as io] - [clojure.string :as string]) + [clojure.string :as string] + [clojure.tools.logging :as log]) (:import (java.io ByteArrayInputStream File) (java.util Arrays) @@ -50,9 +52,24 @@ (println "No additional properties found.\n")) settings)) -(defn ^:private build-engine [dependency-check-properties clj-watson-properties] - (let [settings (create-settings dependency-check-properties clj-watson-properties)] - (Engine. settings))) +(defn ^:private validate-settings + "Validate settings, logging any findings. + Returns {:exit } when app should exit with exit , else nil" + [settings {:keys [run-without-nvd-api-key] :as opts}] + (when (not (or (.getString settings "nvd.api.key") (System/getProperty "nvd.api.key"))) + (if run-without-nvd-api-key + (log/warn (format (str "We cannot recommend running without an nvd.api.key specified.\n" + " You have opted to ignore this advice via the %s option.\n" + " Expect slow NVD data updates and downloads.\n") + (cli-spec/styled-long-opt :run-without-nvd-api-key opts))) + (do (log/fatal (format (str "We cannot recommend running without an nvd.api.key specified.\n" + " If you insist, rerun with the %s option, but be warned\n" + " that you will experience slow NVD data updates and downloads.") + (cli-spec/styled-long-opt :run-without-nvd-api-key opts))) + {:exit 1})))) + +(defn ^:private build-engine [settings] + (Engine. settings)) (defn ^:private clojure-file? [dependency-path] (string/ends-with? dependency-path ".jar")) @@ -68,9 +85,12 @@ engine) (defn start! - [dependencies dependency-check-properties clj-watson-properties] - (with-open [engine (build-engine dependency-check-properties clj-watson-properties)] - (-> engine - (scan-jars dependencies) - (.getDependencies) - (Arrays/asList)))) + [dependencies dependency-check-properties clj-watson-properties opts] + (let [settings (create-settings dependency-check-properties clj-watson-properties)] + (when-let [{:keys [exit]} (validate-settings settings opts)] + (System/exit exit)) + (with-open [engine (build-engine settings)] + (-> engine + (scan-jars dependencies) + (.getDependencies) + (Arrays/asList))))) diff --git a/src/clj_watson/entrypoint.clj b/src/clj_watson/entrypoint.clj index ddd6567..f6089c3 100644 --- a/src/clj_watson/entrypoint.clj +++ b/src/clj_watson/entrypoint.clj @@ -26,14 +26,15 @@ vulnerable-dependencies))) (defmethod scan* :dependency-check [{:keys [deps-edn-path suggest-fix aliases - dependency-check-properties clj-watson-properties]}] + dependency-check-properties clj-watson-properties] :as opts}] ;; dependency-check uses Apache Commons JCS, ask it to use log4j2 to allow us to configure its noisy logging (System/setProperty "jcs.logSystem" "log4j2") (let [{:keys [deps dependencies]} (controller.deps/parse deps-edn-path aliases) repositories (select-keys deps [:mvn/repos]) scanned-dependencies (controller.dc.scanner/start! dependencies dependency-check-properties - clj-watson-properties) + clj-watson-properties + opts) vulnerable-dependencies (controller.dc.vulnerability/extract scanned-dependencies dependencies repositories)] (if suggest-fix (controller.remediate/scan vulnerable-dependencies deps)