Add: standalone plugin for deprecating VTs and add to pyproject.toml

pyproject.toml

@@ -85,6 +85,7 @@ troubadix-changed-packages = 'troubadix.standalone_plugins.changed_packages.chan
 troubadix-changed-cves = 'troubadix.standalone_plugins.changed_cves:main'
 troubadix-allowed-rev-diff = 'troubadix.standalone_plugins.allowed_rev_diff:main'
 troubadix-file-extensions = 'troubadix.standalone_plugins.file_extensions:main'
+troubadix-deprecate-vts = 'troubadix.standalone_plugins.deprecate_vts:main'
 
 [build-system]
 requires = ["poetry-core>=1.0.0"]

tests/standalone_plugins/test_deprecate_vts.py

@@ -0,0 +1,74 @@
+# SPDX-License-Identifier: GPL-3.0-or-later
+# SPDX-FileCopyrightText: 2024 Greenbone AG
+# pylint: disable=line-too-long
+import unittest
+from pathlib import Path
+from tests.plugins import TemporaryDirectory
+
+from troubadix.standalone_plugins.deprecate_vts import (
+    deprecate, parse_args, DeprecatedFile, get_summary,
+    finalize_content, update_summary
+)
+
+
+class ParseArgsTestCase(unittest.TestCase):
+    def test_parse_args(self):
+        testfile = "testfile.nasl"
+        output_path = "attic/"
+        input_path = "nasl/common"
+        oid_mapping_file = "oid_file.py"
+
+        args = parse_args(["--file", str(testfile), "--output-path", output_path, "--oid-mapping-path", oid_mapping_file, "--input-path", input_path])
+        self.assertEqual(args.file, Path(testfile))
+        self.assertEqual(args.output_path, output_path)
+        self.assertEqual(args.oid_mapping_path, oid_mapping_file)
+        self.assertEqual(args.input_path, input_path)
+
+
+NASL_CONTENT = ('...if(description)\n{\n  script_oid("1.3.6.1.4.1.25623.1.0.910673");'
+                '\n  script_version("2024-03-12T14:15:13+0000");'
+                '\n  script_name("RedHat: Security Advisory for gd (RHSA-2020:5443-01)");'         
+                '\n  script_family("Red Hat Local Security Checks");\n  script_dependencies("gather-package-list.nasl");'
+                '\n  script_mandatory_keys("ssh/login/rhel", "ssh/login/rpms", re:"ssh/login/release=RHENT_7");'
+                '\n\n  script_xref(name:"RHSA", value:"2020:5443-01");\n  script_xref(name:"URL", value:"https://www.redhat.com/archives/rhsa-announce/2020-December/msg00044.html");'
+                '\n\n  script_tag(name:"summary", value:"The remote host is missing an update for the \'gd\'\n  package(s) announced via the RHSA-2020:5443-01 advisory.");'
+                '\n\n  exit(0);\n}\n\ninclude("revisions-lib.inc");\ninclude("pkg-lib-rpm.inc");\n\nrelease = rpm_get_ssh_release();\nif(!release)\n  exit(0);\n\nres = "";\nreport = "";\n\nif(release == "RHENT_7") {\n\n  if(!isnull(res = isrpmvuln(pkg:"gd", rpm:"gd~2.0.35~27.el7_9", rls:"RHENT_7"))) {\n    report += res;\n  }\n\n  if(!isnull(res = isrpmvuln(pkg:"gd-debuginfo", rpm:"gd-debuginfo~2.0.35~27.el7_9", rls:"RHENT_7"))) {\n    report += res;\n  }\n\n  if(report != "") {\n    security_message(data:report);\n  } else if(__pkg_match) {\n    exit(99);\n  }\n  exit(0);\n}\n\nexit(0);')
+
+
+class DeprecateVTsTestCase(unittest.TestCase):
+    def test_deprecate(self):
+        with TemporaryDirectory() as out_dir, TemporaryDirectory() as in_dir:
+            testfile1 = in_dir / "testfile1.nasl"
+            testfile1.write_text(NASL_CONTENT, encoding="utf8")
+
+            testfile2 = out_dir/"testfile1.nasl"
+            testfile2.touch()
+
+            to_deprecate = [DeprecatedFile(name="testfile1.nasl", full_path=testfile1,
+                                           content=NASL_CONTENT)]
+            deprecate(out_dir, to_deprecate)
+
+            result = testfile2.read_text(encoding="utf8")
+            self.assertNotIn(result, 'script_mandatory_keys')
+            self.assertNotIn(result, 'script_dependencies')
+            self.assertNotIn(result, 'include("revisions-lib.inc");')
+            assert "This VT has been deprecated." in result
+
+    def test_get_summary(self):
+        result = get_summary(NASL_CONTENT)
+        expected = ("The remote host is missing an update for the \'gd\'\n  package(s) announced "
+                    "via the RHSA-2020:5443-01 advisory.")
+        self.assertEqual(result, expected)
+
+    def test_finalize_content(self):
+        result = finalize_content(NASL_CONTENT)
+        expected = ('...if(description)\n{\n  script_oid("1.3.6.1.4.1.25623.1.0.910673");\n  '
+                    'script_version("2024-03-12T14:15:13+0000");\n  script_name("RedHat: Security Advisory for gd (RHSA-2020:5443-01)");\n  script_family("Red Hat Local Security Checks");\n  script_dependencies("gather-package-list.nasl");\n  script_mandatory_keys("ssh/login/rhel", "ssh/login/rpms", re:"ssh/login/release=RHENT_7");\n\n  script_xref(name:"RHSA", value:"2020:5443-01");\n  script_xref(name:"URL", value:"https://www.redhat.com/archives/rhsa-announce/2020-December/msg00044.html");\n\n  script_tag(name:"summary", value:"The remote host is missing an update for the \'gd\'\n  package(s) announced via the RHSA-2020:5443-01 advisory.");\n\n  script_tag(name: \'deprecated\', value: TRUE);\n\nexit(0);\n}\n\nexit(66);\n')
+        self.assertEqual(result, expected)
+
+    def test_update_summary_no_oid_match(self):
+        file = DeprecatedFile(name="testfile.nasl", full_path=Path("dir/testfile.nasl"), content=NASL_CONTENT)
+        result = update_summary(file)
+        expected = ("This VT has been deprecated.The remote host is missing an update for the 'gd'\n "
+                    " package(s) announced via the RHSA-2020:5443-01 advisory.")
+        self.assertEqual(result, expected)

troubadix/helper/patterns.py

@@ -192,6 +192,7 @@ class SpecialScriptTag(Enum):
     REQUIRE_KEYS = "require_keys"
     REQUIRE_PORTS = "require_ports"
     REQUIRE_UDP_PORTS = "require_udp_ports"
+    SET_KB_ITEM = "set_kb_item"
     VERSION = "version"  # script_version("YYYY-MM-DDTHH:mm:ss+0000");
     XREF = "xref"
 
@@ -232,6 +233,7 @@ def _get_special_script_tag_pattern(
     SpecialScriptTag.REQUIRE_UDP_PORTS: __PORT_VALUE,
     SpecialScriptTag.XREF: r"name:\"(?P<ref_type>[\w\s]+)\","
     r" value:\"(?P<ref>[\w/:= %._\-\?]+)\"",
+    SpecialScriptTag.SET_KB_ITEM: r"set_kb_item\(.+\);"
 }
 
 

troubadix/standalone_plugins/deprecate_vts.py

@@ -0,0 +1,229 @@
+# SPDX-License-Identifier: GPL-3.0-or-later
+# SPDX-FileCopyrightText: 2024 Greenbone AG
+
+import os
+import re
+import sys
+from argparse import ArgumentParser, Namespace
+from dataclasses import dataclass
+from pathlib import Path
+from typing import Iterable, Optional
+
+from troubadix.argparser import file_type
+from troubadix.helper.patterns import (get_special_script_tag_pattern, get_script_tag_pattern,
+                                       ScriptTag, SpecialScriptTag)
+
+
+class PathException(Exception):
+    pass
+
+
+@dataclass
+class DeprecatedFile:
+    name: str
+    full_path: Path
+    content: str
+
+
+# CHANGE AS NEEDED
+FILENAME_REGEX = re.compile(r"gb_rhsa_2020")
+
+
+def update_summary(file: DeprecatedFile, oid_mapping_path: Path = None) -> str:
+    """Update the summary of the nasl script by adding the information
+    that the script has been deprecated, and if possible, the oid of
+    the new notus script replacing it.
+
+    Args:
+        file: DeprecatedFile object containing the content of the VT
+        oid_mapping_path (optional): The path to the file that contains a mapping of
+            old oids to new oids (see notus-generator transition layer)
+    Returns:
+        The updated content of the file
+    """
+    if oid_mapping_path:
+        oid = match_oid(file.content, oid_mapping_path)
+        deprecate_text = f"This VT has been replaced by the new VT: {oid}. "
+    else:
+        deprecate_text = "This VT has been deprecated."
+
+    summary = get_summary(file.content)
+    if summary:
+        file.content = deprecate_text + summary
+    else:
+        print(f"No summary in: {file.name}")
+
+    return file.content
+
+
+def match_oid(content: str, oid_mapping_path: Path) -> str:
+    """Find the new Notus oid that has been mapped to the old
+    OID, so we can add this to the deprecation note.
+    """
+    pattern = get_special_script_tag_pattern(SpecialScriptTag.OID)
+    match_oid = re.search(pattern, content)
+    old_oid = match_oid.group(1)
+
+    # needs improvement
+    sys.path.append(oid_mapping_path)
+    from redhat import mapping
+    reverse_mapping = dict((v, k) for k, v in mapping.items())
+    new_oid = reverse_mapping.get(old_oid)
+    return new_oid
+
+def finalize_content(content: str) -> str:
+    """Update the content field of the nasl script by adding the
+    deprecated tag and removing the extra content."""
+    content_to_keep = content.split("exit(0);")[0]
+    return content_to_keep + ("script_tag(name: 'deprecated', value: TRUE);"
+                              "\n\nexit(0);\n}\n\nexit(66);\n")
+
+
+def get_files(dir_path: Path = None, file: Path = None) -> list[DeprecatedFile]:
+    """Create a list of DeprecatedFile objects
+
+    Args:
+        dir_path (optional): The path to the directory with the files to be deprecated
+        file (optional): The path to the single file to be deprecated.
+
+    Returns:
+        List of DeprecatedFile objects
+    """
+    to_deprecate = []
+    if file and re.match(FILENAME_REGEX, file.name):
+        to_deprecate.append(
+            DeprecatedFile(
+                file.name,
+                file.absolute(),
+                file.open("r", encoding="latin-1").read(),
+            )
+        )
+    else:
+        valid_files = [file for file in dir_path.glob("**/*") if re.match(FILENAME_REGEX, file.name)]
+        for file in valid_files:
+            to_deprecate.append(
+                DeprecatedFile(
+                    file.name,
+                    file.absolute(),
+                    file.open("r", encoding="latin-1").read(),
+                )
+            )
+    return to_deprecate
+
+
+def get_summary(content: str) -> Optional[str]:
+    """Extract the summary from the nasl script"""
+    pattern = get_script_tag_pattern(ScriptTag.SUMMARY)
+    if match_summary := re.search(pattern, content):
+        value = match_summary.group().split('value:"')[1]
+        return value.replace("\");", "")
+    return None
+
+
+def deprecate(output_path: Path, to_deprecate: list[DeprecatedFile],
+              oid_mapping_path: Path = None) -> None:
+    """Deprecate the selected VTs by removing unnecessary keys, updating the
+    summary, and adding the deprecated tag.
+
+    Args:
+        output_path: the directory where the deprecated VTs should be written
+            to, i.e. "attic"
+        to_deprecate: the list of files to be deprecated
+        oid_mapping_path (optional) : the path to the file where the old oids have been
+            mapped to the new oids (see "transition_layer" in notus-generator).
+    """
+    output_path.mkdir(parents=True, exist_ok=True)
+    for file in to_deprecate:
+        kb_pattern = get_special_script_tag_pattern(SpecialScriptTag.SET_KB_ITEM)
+        items = re.search(kb_pattern, file.content)
+        if items:
+            print(f"Unable to deprecate {file.name}. There are still KB keys remaining.")
+            continue
+        file.content = update_summary(file, oid_mapping_path)
+        file.content = finalize_content(file.content)
+
+        # Drop any unnecessary script tags like script_dependencies(), script_require_udp_ports()
+        # or script_mandatory_keys()
+        tags_to_remove = list()
+        dependencies = re.search(get_special_script_tag_pattern(SpecialScriptTag.DEPENDENCIES), file.content)
+        if dependencies:
+            tags_to_remove += dependencies.group()
+
+        udp = re.search(get_special_script_tag_pattern(SpecialScriptTag.REQUIRE_UDP_PORTS), file.content)
+        if udp:
+            tags_to_remove += udp.group()
+
+        man_keys = re.search(get_special_script_tag_pattern(SpecialScriptTag.MANDATORY_KEYS), file.content)
+        if man_keys:
+            tags_to_remove += man_keys.group()
+
+        for tag in tags_to_remove:
+            file.content = file.content.replace(tag + "\n", "")
+
+        os.rename(file.full_path, output_path / file.name)
+
+        with open(output_path / file.name, "w", encoding="latin-1") as f:
+            f.write(file.content)
+            f.truncate()
+
+
+def parse_args(args: Iterable[str] = None) -> Namespace:
+    parser = ArgumentParser(description="Deprecate VTs")
+    parser.add_argument(
+        "-o",
+        "--output-path",
+        metavar="<output_path>",
+        type=str,
+        required=True,
+        help="Path where the deprecated files should be written to.",
+    )
+    parser.add_argument(
+        "-f",
+        "--file",
+        metavar="<file>",
+        nargs='?',
+        default=None,
+        type=file_type,
+        help="single file to deprecate",
+    )
+    parser.add_argument(
+        "-i",
+        "--input-path",
+        metavar="<input_path>",
+        nargs='?',
+        default=None,
+        type=str,
+        help="Path to the existing nasl scripts",
+    )
+    parser.add_argument(
+        "-m",
+        "--oid-mapping-path",
+        metavar="<oid_mapping_path>",
+        nargs='?',
+        default=None,
+        type=str,
+        help="Path to the oid mapping file",
+    )
+    return parser.parse_args(args)
+
+
+def main():
+    args = parse_args()
+    output_path = Path(args.output_path)
+    input_path = Path(args.input_path) if args.input_path else None
+    oid_mapping_path = args.oid_mapping_path if args.oid_mapping_path else None
+    single_file = Path(args.file) if args.file else None
+
+    if not input_path and not single_file:
+        raise PathException("Please provide either the path to a single file or a directory.")
+
+    if not input_path.is_dir():
+        raise PathException("Input path is not a directory.")
+
+    to_deprecate = get_files(input_path, single_file)
+
+    deprecate(output_path, to_deprecate, oid_mapping_path)
+
+
+if __name__ == "__main__":
+    main()