diff --git a/README.md b/README.md index b715d61..51570d3 100644 --- a/README.md +++ b/README.md @@ -136,6 +136,11 @@ $ nix eval -f '' 'wget.outPath' By default `sbomnix` scans the given target and generates an SBOM including the runtime dependencies. Notice: determining the target runtime dependencies in Nix requires building the target. ```bash +# Target can be specified with flakeref too, e.g.: +# sbomnix . +# sbomnix github:tiiuae/sbomnix +# sbomnix nixpkgs#wget +# Ref: https://nixos.org/manual/nix/stable/command-ref/new-cli/nix3-flake.html#flake-references $ sbomnix /nix/store/8nbv1drmvh588pwiwsxa47iprzlgwx6j-wget-1.21.3 ... INFO Wrote: sbom.cdx.json diff --git a/doc/nix_outdated.md b/doc/nix_outdated.md index 014d459..99b6449 100644 --- a/doc/nix_outdated.md +++ b/doc/nix_outdated.md @@ -23,11 +23,16 @@ $ nix eval -f '' 'git.outPath' ``` # nix_outdated -[`nix_outdated`](../src/nixupdate/nix_outdated.py) is a command line tool to list outdated nix dependencies for given target nix out path. By default, the script outputs runtime dependencies for the given nix out path that appear outdated in nixpkgs 'nix_unstable' channel - the list of output packages would potentially need a PR to update the package in nixpkgs to the package's latest upstream release version specified in the output table column 'version_upstream'. The list of output packages is in priority order based on how many other packages depend on the potentially outdated package. +[`nix_outdated`](../src/nixupdate/nix_outdated.py) is a command line tool to list outdated nix dependencies for given target nix out path or flakeref. By default, the script outputs runtime dependencies for the given target that appear outdated in nixpkgs 'nix_unstable' channel - the list of output packages would potentially need a PR to update the package in nixpkgs to the package's latest upstream release version specified in the output table column 'version_upstream'. The list of output packages is in priority order based on how many other packages depend on the potentially outdated package. Below command finds `git` runtime dependencies that would have an update in the package's upstream repository based on repology, and the latest release version is not available in nix unstable: ```bash +# Target can be specified with flakeref too, e.g.: +# nix_outdated . +# nix_outdated github:tiiuae/sbomnix +# nix_outdated nixpkgs#git +# Ref: https://nixos.org/manual/nix/stable/command-ref/new-cli/nix3-flake.html#flake-references $ nix_outdated /nix/store/2853v0cidl7jww2hs1mlkg0i372mk368-git-2.39.2 INFO Generating SBOM for target '/nix/store/2853v0cidl7jww2hs1mlkg0i372mk368-git-2.39.2' INFO Loading runtime dependencies referenced by '/nix/store/2853v0cidl7jww2hs1mlkg0i372mk368-git-2.39.2' diff --git a/doc/nixgraph.md b/doc/nixgraph.md index 9838a74..da5ea7b 100644 --- a/doc/nixgraph.md +++ b/doc/nixgraph.md @@ -42,6 +42,11 @@ $ nix eval -f '' 'wget.outPath' #### Example: package runtime dependencies ```bash +# Target can be specified with flakeref too, e.g.: +# nixgraph . +# nixgraph github:tiiuae/sbomnix +# nixgraph nixpkgs#wget +# Ref: https://nixos.org/manual/nix/stable/command-ref/new-cli/nix3-flake.html#flake-references $ nixgraph /nix/store/8nbv1drmvh588pwiwsxa47iprzlgwx6j-wget-1.21.3 INFO Loading runtime dependencies referenced by '/nix/store/8nbv1drmvh588pwiwsxa47iprzlgwx6j-wget-1.21.3' diff --git a/doc/vulnxscan.md b/doc/vulnxscan.md index a66f209..9552f20 100644 --- a/doc/vulnxscan.md +++ b/doc/vulnxscan.md @@ -67,6 +67,11 @@ Vulnix matches vulnerabilities based on [heuristic](https://github.com/nix-commu This example shows how to use `vulnxscan` to summarize vulnerabilities impacting the given target or any of its runtime dependencies. ```bash +# Target can be specified with flakeref too, e.g.: +# vulnxscan . +# vulnxscan github:tiiuae/sbomnix +# vulnxscan nixpkgs#git +# Ref: https://nixos.org/manual/nix/stable/command-ref/new-cli/nix3-flake.html#flake-references $ vulnxscan /nix/store/ay9sn71cssl4wd7s6bd8xah0zcwqiq2q-git-2.41.0.drv INFO Generating SBOM for target '/nix/store/ay9sn71cssl4wd7s6bd8xah0zcwqiq2q-git-2.41.0.drv' diff --git a/src/common/utils.py b/src/common/utils.py index 8f332bb..0257613 100644 --- a/src/common/utils.py +++ b/src/common/utils.py @@ -160,12 +160,13 @@ def exit_unless_command_exists(name): def exit_unless_nix_artifact(path, force_realise=False): """ - Exit with error if `path` is not a nix artifact. If `force_realize` is True, - run the nix-store-query command with `--force-realize` realising the `path` + Exit with error if `path` is not a nix artifact. If `force_realise` is True, + run the nix-store-query command with `--force-realise` realising the `path` argument before running query. """ LOG.debug("force_realize: %s", force_realise) if force_realise: + LOG.info("Try force-realising store-path '%s'", path) cmd = ["nix-store", "-qf", path] else: cmd = ["nix-store", "-q", path] @@ -177,6 +178,30 @@ def exit_unless_nix_artifact(path, force_realise=False): sys.exit(1) +def try_resolve_flakeref(flakeref, force_realise=False): + """ + Resolve flakeref to out-path, force-realising the output if `force_realise` + is True. Returns resolved path if flakeref can be resolved to out-path, + otherwise, returns None. + """ + LOG.info("Evaluating '%s'", flakeref) + cmd = f"nix eval --raw {flakeref}" + ret = exec_cmd(cmd.split(), raise_on_error=False) + if not ret: + LOG.debug("not a flakeref: '%s'", flakeref) + return None + nixpath = ret.stdout + LOG.debug("flakeref='%s' maps to path='%s'", flakeref, nixpath) + if not force_realise: + return nixpath + LOG.info("Try force-realising flakeref '%s'", flakeref) + cmd = f"nix build --no-link {flakeref}" + ret = exec_cmd(cmd.split(), raise_on_error=False, return_error=True) + if not ret: + LOG.fatal("Failed force_realising %s: %s", flakeref, ret.stderr) + return nixpath + + def regex_match(regex, string): """Return True if regex matches string""" if not regex or not string: diff --git a/src/nixgraph/main.py b/src/nixgraph/main.py index f828586..837a6d7 100755 --- a/src/nixgraph/main.py +++ b/src/nixgraph/main.py @@ -14,6 +14,7 @@ get_py_pkg_version, check_positive, exit_unless_nix_artifact, + try_resolve_flakeref, ) ############################################################################### @@ -25,8 +26,11 @@ def getargs(): epil = "Example: nixgraph /path/to/derivation.drv " parser = argparse.ArgumentParser(description=desc, epilog=epil) - helps = "Path to nix artifact, e.g.: derivation file or nix output path" - parser.add_argument("NIX_PATH", help=helps, type=pathlib.Path) + helps = ( + "Target nix store path (e.g. derivation file or nix output path) or flakeref" + ) + parser.add_argument("NIXREF", help=helps, type=str) + parser.add_argument("--version", action="version", version=get_py_pkg_version()) helps = "Scan buildtime dependencies instead of runtime dependencies" @@ -81,9 +85,11 @@ def main(): """main entry point""" args = getargs() set_log_verbosity(args.verbose) - target_path = args.NIX_PATH.resolve().as_posix() runtime = args.buildtime is False - exit_unless_nix_artifact(target_path, force_realise=runtime) + target_path = try_resolve_flakeref(args.NIXREF, force_realise=runtime) + if not target_path: + target_path = pathlib.Path(args.NIXREF).resolve().as_posix() + exit_unless_nix_artifact(args.NIXREF, force_realise=runtime) deps = NixDependencies(target_path, args.buildtime) deps.graph(args) diff --git a/src/nixupdate/nix_outdated.py b/src/nixupdate/nix_outdated.py index 595f82f..b441734 100755 --- a/src/nixupdate/nix_outdated.py +++ b/src/nixupdate/nix_outdated.py @@ -26,6 +26,7 @@ df_to_csv_file, nix_to_repology_pkg_name, exit_unless_nix_artifact, + try_resolve_flakeref, ) ############################################################################### @@ -34,9 +35,9 @@ def getargs(): """Parse command line arguments""" desc = ( - "Command line tool to list outdated nix dependencies for nix out path " - "NIXPATH. By default, the script outputs runtime dependencies of " - "NIXPATH that appear outdated in nixpkgs 'nix_unstable' channel - the " + "Command line tool to list outdated nix dependencies for NIXREF. " + "By default, the script outputs runtime dependencies of " + "NIXREF that appear outdated in nixpkgs 'nix_unstable' channel - the " "list of output packages would potentially need a PR to update the " "package in nixpkgs to the latest upstream release version specified " "in the output table column 'version_upstream'. " @@ -44,11 +45,13 @@ def getargs(): "order based on how many other packages depend on the potentially " "outdated package." ) - epil = f"Example: ./{os.path.basename(__file__)} '/nix/target/out/path'" + epil = f"Example: ./{os.path.basename(__file__)} '/nix/path/or/flakeref'" parser = ArgumentParser(description=desc, epilog=epil) # Arguments that specify the target: - helps = "Target nix out path" - parser.add_argument("NIXPATH", help=helps, type=pathlib.Path) + helps = ( + "Target nix store path (e.g. derivation file or nix output path) or flakeref" + ) + parser.add_argument("NIXREF", help=helps, type=str) # Other arguments: helps = ( "Include locally outdated dependencies to the output. " @@ -77,7 +80,7 @@ def getargs(): def _generate_sbom(target_path, buildtime=False): LOG.info("Generating SBOM for target '%s'", target_path) - sbomdb = SbomDb(target_path, buildtime) + sbomdb = SbomDb(target_path, buildtime, include_meta=False) prefix = "nixdeps_" suffix = ".cdx.json" with NamedTemporaryFile(delete=False, prefix=prefix, suffix=suffix) as f: @@ -250,13 +253,14 @@ def main(): """main entry point""" args = getargs() set_log_verbosity(args.verbose) - target_path_abs = args.NIXPATH.resolve().as_posix() runtime = args.buildtime is False + target_path = try_resolve_flakeref(args.NIXREF, force_realise=runtime) + if not target_path: + target_path = pathlib.Path(args.NIXREF).resolve().as_posix() + exit_unless_nix_artifact(args.NIXREF, force_realise=runtime) dtype = "runtime" if runtime else "buildtime" - LOG.info("Checking %s dependencies referenced by '%s'", dtype, target_path_abs) - exit_unless_nix_artifact(target_path_abs, force_realise=runtime) - - sbom_path = _generate_sbom(target_path_abs, args.buildtime) + LOG.info("Checking %s dependencies referenced by '%s'", dtype, target_path) + sbom_path = _generate_sbom(target_path, args.buildtime) LOG.debug("Using SBOM '%s'", sbom_path) df_repology = _run_repology_cli(sbom_path) @@ -265,7 +269,7 @@ def main(): df_log(df_repology, LOG_SPAM) if not args.buildtime: - nix_visualize_out = _run_nix_visualize(target_path_abs) + nix_visualize_out = _run_nix_visualize(target_path) LOG.debug("Using nix-visualize out: '%s'", nix_visualize_out) df_nix_visualize = _nix_visualize_csv_to_df(nix_visualize_out) df_log(df_nix_visualize, LOG_SPAM) diff --git a/src/sbomnix/cpe.py b/src/sbomnix/cpe.py index 0d3d221..12d47f0 100644 --- a/src/sbomnix/cpe.py +++ b/src/sbomnix/cpe.py @@ -2,7 +2,7 @@ # # SPDX-License-Identifier: Apache-2.0 -# pylint: disable=invalid-name, protected-access, too-few-public-methods +# pylint: disable=invalid-name, too-few-public-methods """ Generate CPE (Common Platform Enumeration) identifiers""" @@ -27,18 +27,9 @@ ############################################################################### -def CPE(): - """Return CPE instance""" - if _CPE._instance is None: - _CPE._instance = _CPE() - return _CPE._instance - - -class _CPE: +class CPE: """Generate Common Platform Enumeration identifiers""" - _instance = None - def __init__(self): self.cache = DataFrameDiskCache(cache_dir_path=DFCACHE_PATH) self.df_cpedict = self.cache.get(_CPE_CSV_URL) diff --git a/src/sbomnix/derivation.py b/src/sbomnix/derivation.py index fc9a2dd..2d1e213 100644 --- a/src/sbomnix/derivation.py +++ b/src/sbomnix/derivation.py @@ -12,7 +12,6 @@ import json import bisect from packageurl import PackageURL -from sbomnix.cpe import CPE from common.utils import LOG, LOG_SPAM @@ -83,7 +82,6 @@ def __init__( self.cpe = "" self.purl = "" if self.pname != "source": - self.cpe = CPE().generate(self.pname, self.version) self.purl = str( PackageURL(type="nix", name=self.pname, version=self.version) ) @@ -92,6 +90,11 @@ def __init__( def __repr__(self): return f"" + def set_cpe(self, cpe_generator): + """Generate cpe identifier""" + if self.pname != "source" and cpe_generator is not None: + self.cpe = cpe_generator.generate(self.pname, self.version) + def add_output_path(self, path): """Add an output path to derivation""" if path not in self.outputs and path != self.store_path: diff --git a/src/sbomnix/main.py b/src/sbomnix/main.py index 8045db5..0ef1698 100755 --- a/src/sbomnix/main.py +++ b/src/sbomnix/main.py @@ -10,12 +10,11 @@ import pathlib from sbomnix.sbomdb import SbomDb from common.utils import ( - LOG, set_log_verbosity, check_positive, get_py_pkg_version, exit_unless_nix_artifact, - exec_cmd, + try_resolve_flakeref, ) ############################################################################### @@ -31,7 +30,9 @@ def getargs(): epil = "Example: sbomnix /nix/store/path/or/flakeref" parser = argparse.ArgumentParser(description=desc, epilog=epil) - helps = "Nix store path (e.g. derivation file or nix output path) or flakeref" + helps = ( + "Target nix store path (e.g. derivation file or nix output path) or flakeref" + ) parser.add_argument("NIXREF", help=helps, type=str) helps = "Scan buildtime dependencies instead of runtime dependencies" parser.add_argument("--buildtime", help=helps, action="store_true") @@ -62,25 +63,6 @@ def getargs(): ################################################################################ -def try_resolve_flakeref(flakeref, force_realise): - """Resolve flakeref to out-path""" - LOG.debug("") - cmd = f"nix eval --raw {flakeref}" - ret = exec_cmd(cmd.split(), raise_on_error=False) - if not ret: - LOG.debug("not a flakeref: '%s'", flakeref) - return None - nixpath = ret.stdout - LOG.debug("nixpath=%s", nixpath) - if not force_realise: - return nixpath - cmd = f"nix build --no-link {flakeref}" - ret = exec_cmd(cmd.split(), raise_on_error=False, return_error=True) - if not ret: - LOG.fatal("Failed force_realising %s: %s", flakeref, ret.stderr) - return nixpath - - def main(): """main entry point""" args = getargs() @@ -90,7 +72,6 @@ def main(): target_path = try_resolve_flakeref(args.NIXREF, force_realise=runtime) if target_path: flakeref = args.NIXREF - LOG.debug("flakeref='%s' maps to path='%s'", flakeref, target_path) else: target_path = pathlib.Path(args.NIXREF).resolve().as_posix() exit_unless_nix_artifact(args.NIXREF, force_realise=runtime) diff --git a/src/sbomnix/nix.py b/src/sbomnix/nix.py index 1e872fc..77f0ec7 100644 --- a/src/sbomnix/nix.py +++ b/src/sbomnix/nix.py @@ -12,6 +12,7 @@ from common.utils import LOG, LOG_SPAM, exec_cmd from sbomnix.derivation import load +from sbomnix.cpe import CPE ############################################################################### @@ -22,6 +23,7 @@ class Store: def __init__(self, buildtime=False): self.buildtime = buildtime self.derivations = {} + self.cpe_generator = CPE() def _add_cached(self, path, drv): LOG.log(LOG_SPAM, "caching path - %s:%s", path, drv) @@ -51,6 +53,7 @@ def _update(self, drv_path, nixpath=None): drv_obj = self._get_cached(drv_path) if not drv_obj: drv_obj = load(drv_path) + drv_obj.set_cpe(self.cpe_generator) self._add_cached(drv_path, drv=drv_obj) assert drv_obj.store_path == drv_path, f"unexpected drv_path: {drv_path}" if nixpath: diff --git a/src/sbomnix/sbomdb.py b/src/sbomnix/sbomdb.py index 49c7fd1..9d6d474 100644 --- a/src/sbomnix/sbomdb.py +++ b/src/sbomnix/sbomdb.py @@ -4,7 +4,7 @@ # # SPDX-License-Identifier: Apache-2.0 -# pylint: disable=invalid-name, too-many-instance-attributes +# pylint: disable=invalid-name, too-many-instance-attributes, too-many-arguments """ Module for generating SBOMs in various formats """ @@ -28,7 +28,9 @@ class SbomDb: """Generates SBOMs in various formats""" - def __init__(self, nix_path, buildtime=False, depth=None, flakeref=None): + def __init__( + self, nix_path, buildtime=False, depth=None, flakeref=None, include_meta=True + ): # self.uid specifies the attribute that SbomDb uses as unique # identifier for the sbom components. See the column names in # self.df_sbomdb (sbom.csv) for a list of all components' attributes. @@ -41,7 +43,8 @@ def __init__(self, nix_path, buildtime=False, depth=None, flakeref=None): self.df_sbomdb = None self.df_sbomdb_outputs_exploded = None self.flakeref = flakeref - self._init_sbomdb() + self.meta = None + self._init_sbomdb(include_meta) self.uuid = uuid.uuid4() self.sbom_type = "runtime_and_buildtime" if not self.buildtime: @@ -68,7 +71,7 @@ def _get_dependencies_df(self, nix_dependencies): LOG.debug("Reading all dependencies") return nix_dependencies.to_dataframe() - def _init_sbomdb(self): + def _init_sbomdb(self, include_meta): """Initialize self.df_sbomdb""" if self.df_deps is None or self.df_deps.empty: # No dependencies, so the only component in the sbom @@ -85,7 +88,8 @@ def _init_sbomdb(self): store.add_path(path) self.df_sbomdb = store.to_dataframe() # Join with meta information - self._sbomdb_join_meta() + if include_meta: + self._sbomdb_join_meta() # Clean, drop duplicates, sort self.df_sbomdb.replace(np.nan, "", regex=True, inplace=True) self.df_sbomdb.drop_duplicates(subset=[self.uid], keep="first", inplace=True) @@ -94,11 +98,11 @@ def _init_sbomdb(self): def _sbomdb_join_meta(self): """Join self.df_sbomdb with meta information""" - meta = Meta() + self.meta = Meta() if self.flakeref: - df_meta = meta.get_nixpkgs_meta(self.flakeref) + df_meta = self.meta.get_nixpkgs_meta(self.flakeref) else: - df_meta = meta.get_nixpkgs_meta() + df_meta = self.meta.get_nixpkgs_meta() if df_meta is None or df_meta.empty: LOG.warning( "Failed reading nix meta information: " diff --git a/src/vulnxscan/vulnxscan_cli.py b/src/vulnxscan/vulnxscan_cli.py index b65bb67..0916e92 100755 --- a/src/vulnxscan/vulnxscan_cli.py +++ b/src/vulnxscan/vulnxscan_cli.py @@ -41,6 +41,7 @@ df_from_csv_file, df_log, exit_unless_nix_artifact, + try_resolve_flakeref, exit_unless_command_exists, nix_to_repology_pkg_name, parse_version, @@ -57,10 +58,12 @@ def getargs(): "Scan nix artifact or CycloneDX SBOM for vulnerabilities with " "various open-source vulnerability scanners." ) - epil = "Example: ./vulnxscan.py /path/to/nix/out/or/drv" + epil = "Example: ./vulnxscan.py /path/to/nix/out/or/drv/or/flakeref" parser = argparse.ArgumentParser(description=desc, epilog=epil) - helps = "Target derivation path or nix out path" - parser.add_argument("TARGET", help=helps, type=pathlib.Path) + helps = ( + "Target nix store path (e.g. derivation file or nix output path) or flakeref" + ) + parser.add_argument("TARGET", help=helps, type=str) helps = "Set the debug verbosity level between 0-3 (default: --verbose=1)" parser.add_argument("--verbose", help=helps, type=int, default=1) helps = "Path to output file (default: ./vulns.csv)" @@ -736,7 +739,7 @@ def _is_patched(row): def _generate_sbom(target_path, buildtime=False): LOG.info("Generating SBOM for target '%s'", target_path) - sbomdb = SbomDb(target_path, buildtime) + sbomdb = SbomDb(target_path, buildtime, include_meta=False) prefix = "vulnxscan_" cdx_suffix = ".json" csv_suffix = ".csv" @@ -863,23 +866,26 @@ def main(): exit_unless_command_exists("grype") exit_unless_command_exists("vulnix") - target_path_abs = args.TARGET.resolve().as_posix() scanner = VulnScan() if args.sbom: - if not _is_json(target_path_abs): + target_path = pathlib.Path(args.TARGET).resolve().as_posix() + if not _is_json(target_path): LOG.fatal( "Specified sbom target is not a json file: '%s'", str(args.TARGET) ) sys.exit(0) - sbom_cdx_path = target_path_abs + sbom_cdx_path = target_path sbom_csv_path = None else: runtime = args.buildtime is False - exit_unless_nix_artifact(target_path_abs, force_realise=runtime) - sbom_cdx_path, sbom_csv_path = _generate_sbom(target_path_abs, args.buildtime) + target_path = try_resolve_flakeref(args.TARGET, force_realise=runtime) + if not target_path: + target_path = pathlib.Path(args.TARGET).resolve().as_posix() + exit_unless_nix_artifact(args.TARGET, force_realise=runtime) + sbom_cdx_path, sbom_csv_path = _generate_sbom(target_path, args.buildtime) LOG.debug("Using cdx SBOM '%s'", sbom_cdx_path) LOG.debug("Using csv SBOM '%s'", sbom_csv_path) - scanner.scan_vulnix(target_path_abs, args.buildtime) + scanner.scan_vulnix(target_path, args.buildtime) scanner.scan_grype(sbom_cdx_path) scanner.scan_osv(sbom_cdx_path) scanner.report(args, sbom_csv_path)