diff --git a/poetry.lock b/poetry.lock index 19cff5ee..029f3ed4 100644 --- a/poetry.lock +++ b/poetry.lock @@ -984,4 +984,4 @@ test = ["flake8 (>=2.4.0)", "isort (>=4.2.2)", "pytest (>=2.2.3)"] [metadata] lock-version = "2.0" python-versions = "^3.9" -content-hash = "b33fd3f05141b11422493b6ae5665de55fffeed3cb4f6e3bffa7048f514f5f96" +content-hash = "ac5bd99d673cdfcd48512b6f97ab6cf729a39ba89760c33dc7851749f26a7965" diff --git a/pyproject.toml b/pyproject.toml index 9048f4a3..ef7179d9 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -9,9 +9,9 @@ repository = "https://github.com/greenbone/troubadix" homepage = "https://github.com/greenbone/troubadix" # Full list: https://pypi.org/pypi?%3Aaction=list_classifiers -classifiers=[ +classifiers = [ "Development Status :: 4 - Beta", - "License :: OSI Approved :: GNU General Public License v3 or later (GPLv3+)", # pylint: disable=line-too-long + "License :: OSI Approved :: GNU General Public License v3 or later (GPLv3+)", # pylint: disable=line-too-long "Environment :: Console", "Intended Audience :: Developers", "Programming Language :: Python :: 3.9", @@ -21,10 +21,7 @@ classifiers=[ "Topic :: Software Development :: Libraries :: Python Modules", ] -packages = [ - { include = "troubadix"}, - { include = "tests", format = "sdist" }, -] +packages = [{ include = "troubadix" }, { include = "tests", format = "sdist" }] [tool.poetry.dependencies] python = "^3.9" @@ -35,6 +32,7 @@ chardet = ">=4,<6" validators = "0.20.0" gitpython = "^3.1.31" charset-normalizer = "^3.2.0" +tomli = { version = "^2.0.1", python = "<3.11" } [tool.poetry.dev-dependencies] autohooks = ">=21.7.0" diff --git a/tests/plugins/test_http_links_in_tags.py b/tests/plugins/test_http_links_in_tags.py index 0906d41f..b3583999 100644 --- a/tests/plugins/test_http_links_in_tags.py +++ b/tests/plugins/test_http_links_in_tags.py @@ -18,13 +18,48 @@ from pathlib import Path -from troubadix.plugin import LinterError +from troubadix.plugin import ConfigurationError, LinterError from troubadix.plugins.http_links_in_tags import CheckHttpLinksInTags from . import PluginTestCase +BASE_CONFIG = {CheckHttpLinksInTags.name: {"exclusions": []}} + class CheckHttpLinksInTagsTestCase(PluginTestCase): + + def test_validate_config(self): + fake_context = self.create_file_plugin_context() + + valid_plugin_config = { + "exclusions": ["Foo Bar. https://www.website.de/demo"] + } + valid_config = {"check_http_links_in_tags": valid_plugin_config} + + # config extraction and validation is done when the init method + # is called with the optional key config. + plugin = CheckHttpLinksInTags(fake_context, config=valid_config) + + self.assertEqual(plugin.config, valid_plugin_config) + + invalid_config_missing_plugin_key = {} + with self.assertRaises(ConfigurationError) as context: + plugin.extract_plugin_config(invalid_config_missing_plugin_key) + self.assertEqual( + str(context.exception), + "Configuration for plugin 'check_http_links_in_tags' is missing.", + ) + invalid_plugin_config_missing_required_key = {} + with self.assertRaises(ConfigurationError) as context: + plugin.validate_plugin_config( + invalid_plugin_config_missing_required_key + ) + self.assertEqual( + str(context.exception), + "Configuration for plugin 'check_http_links_in_tags' " + "is missing required key: 'exclusions'", + ) + def test_ok(self): path = Path("some/file.nasl") content = ( @@ -33,11 +68,18 @@ def test_ok(self): ' script_tag(name:"solution_type", value:"VendorFix");\n' ' script_tag(name:"solution", value:"meh");\n' 'port = get_app_port_from_cpe_prefix("cpe:/o:foo:bar");\n' + ' script_tag(name:"summary", value:"Foo Bar. ' + 'https://www.website.de/demo");\n' ) + fake_plugin_config = { + "exclusions": ["Foo Bar. https://www.website.de/demo"] + } fake_context = self.create_file_plugin_context( - nasl_file=path, file_content=content + nasl_file=path, + file_content=content, ) - plugin = CheckHttpLinksInTags(fake_context) + plugin = CheckHttpLinksInTags(fake_context, config=BASE_CONFIG) + plugin.config = fake_plugin_config results = list(plugin.run()) @@ -46,7 +88,7 @@ def test_ok(self): def test_exclude_inc_file(self): path = Path("some/file.inc") fake_context = self.create_file_plugin_context(nasl_file=path) - plugin = CheckHttpLinksInTags(fake_context) + plugin = CheckHttpLinksInTags(fake_context, config=BASE_CONFIG) results = list(plugin.run()) @@ -64,7 +106,7 @@ def test_not_ok(self): fake_context = self.create_file_plugin_context( nasl_file=path, file_content=content ) - plugin = CheckHttpLinksInTags(fake_context) + plugin = CheckHttpLinksInTags(fake_context, config=BASE_CONFIG) results = list(plugin.run()) @@ -92,7 +134,7 @@ def test_not_ok2(self): fake_context = self.create_file_plugin_context( nasl_file=path, file_content=content ) - plugin = CheckHttpLinksInTags(fake_context) + plugin = CheckHttpLinksInTags(fake_context, config=BASE_CONFIG) results = list(plugin.run()) @@ -106,66 +148,3 @@ def test_not_ok2(self): 'value:"https://nvd.nist.gov/vuln/detail/CVE-1234");', results[0].message, ) - - def test_http_link_in_tags_ok(self): - testcases = [ - "01. The payloads try to open a connection to www.google.com", - "02. The script attempts to connect to www.google.com", - "03. to retrieve a web page from www.google.com", - "04. Subject: commonName=www.paypal.com", - "05. Terms of use at https://www.verisign.com/rpa", - "06. example.com", - "07. example.org", - "08. www.exam", - "09. sampling the resolution of a name (www.google.com)", - "10. once with 'www.' and once without", - "11. wget http://www.javaop.com/~ron/tmp/nc", - "12. as www.windowsupdate.com. (BZ#506016)", - "13. located at http://sambarserver/session/pagecount.", - "14. http://rest.modx.com", - "15. ftp:// ", - "16. ftp://'", - "17. ftp://)", - "18. ftp.c", - "19. ftp.exe", - "20. using special ftp://", - "21. running ftp.", - "22. ftp. The vulnerability", - "23. 'http://' protocol", - "24. handle properly", - "25. Switch to git+https://", - "26. wget https://compromised-domain.com/important-file", - "27. the https:// scheme", - "28. https://www.phishingtarget.com@evil.com", - "29. 'http://'", - "30. 'https://'", - "31. distributions on ftp.proftpd.org have all been", - "32. information from www.mutt.org:", - "33. According to www.tcpdump.org:", - "34. According to www.kde.org:", - "35. From the www.info-zip.org site:", - # pylint: disable=line-too-long - "36. (www.isg.rhul.ac.uk) for discovering this flaw and Adam Langley and", - "37. Sorry about having to reissue this one -- I pulled it from ftp.gnu.org not", - "38. http://internal-host$1 is still insecure", - "39. from online sources (ftp://, http:// etc.).", - "40. this and https:// and that.", - "41. such as 'http://:80'", - "42. ", - ] - - for testcase in testcases: - self.assertTrue(CheckHttpLinksInTags.check_to_continue(testcase)) - - def test_http_link_in_tags_not_ok(self): - testcases = [ - "The payloads try to open a connection to www.bing.com", - "examplephishing.org", - "located at http://sambdadancinglessions/session/pagecount.", - "fdp:// ", - "Switch to svn+https://", - "greenbone.net", - ] - - for testcase in testcases: - self.assertFalse(CheckHttpLinksInTags.check_to_continue(testcase)) diff --git a/tests/standalone_plugins/test_deprecate_vts.py b/tests/standalone_plugins/test_deprecate_vts.py index 1e5715c7..66709f55 100644 --- a/tests/standalone_plugins/test_deprecate_vts.py +++ b/tests/standalone_plugins/test_deprecate_vts.py @@ -4,88 +4,90 @@ # pylint: disable=protected-access import unittest from pathlib import Path -from tests.plugins import TemporaryDirectory +from tests.plugins import TemporaryDirectory from troubadix.standalone_plugins.deprecate_vts import ( - deprecate, - parse_args, DeprecatedFile, - _get_summary, _finalize_content, - update_summary, - get_files_from_path, + _get_summary, + deprecate, filter_files, + get_files_from_path, + parse_args, + update_summary, ) class ParseArgsTestCase(unittest.TestCase): def test_parse_args(self): testfile = "testfile.nasl" - output_path = "attic/" - reason = "NOTUS" - - args = parse_args( - [ - "--files", - testfile, - "--output-path", - output_path, - "--deprecation-reason", - reason, - ] - ) - self.assertEqual(args.files, [Path(testfile)]) - self.assertEqual(args.output_path, Path(output_path)) - self.assertEqual(args.deprecation_reason, reason) - - def test_mandatory_arg_group_both(self): - testfile = "testfile.nasl" - output_path = "attic/" - input_path = "nasl/common" reason = "NOTUS" - with self.assertRaises(SystemExit): - parse_args( + with TemporaryDirectory() as out_dir: + args = parse_args( [ "--files", testfile, "--output-path", - output_path, - "--input-path", - input_path, + str(out_dir), "--deprecation-reason", reason, ] ) + self.assertEqual(args.files, [Path(testfile)]) + self.assertEqual(args.output_path, out_dir) + self.assertEqual(args.deprecation_reason, reason) + + def test_mandatory_arg_group_both(self): + testfile = "testfile.nasl" + reason = "NOTUS" + + with ( + TemporaryDirectory() as out_dir, + TemporaryDirectory() as in_dir, + ): + with self.assertRaises(SystemExit): + parse_args( + [ + "--files", + testfile, + "--input-path", + str(in_dir), + "--deprecation-reason", + reason, + "--output-path", + str(out_dir), + ] + ) def test_invalid_reason(self): - output_path = "attic/" - input_path = "nasl/common" reason = "foo" - with self.assertRaises(SystemExit): - parse_args( - [ - "--output-path", - output_path, - "--input-path", - input_path, - "--deprecation-reason", - reason, - ] - ) + + with TemporaryDirectory() as out_dir, TemporaryDirectory() as in_dir: + with self.assertRaises(SystemExit): + parse_args( + [ + "--output-path", + str(out_dir), + "--input-path", + str(in_dir), + "--deprecation-reason", + reason, + ] + ) def test_mandatory_arg_group_neither(self): - output_path = "attic/" reason = "NOTUS" - with self.assertRaises(SystemExit): - parse_args( - [ - "--output-path", - output_path, - "--deprecation-reason", - reason, - ] - ) + with TemporaryDirectory() as out_dir: + with self.assertRaises(SystemExit): + parse_args( + [ + "--output-path", + str(out_dir), + "--deprecation-reason", + reason, + ] + ) NASL_CONTENT = ( diff --git a/tests/test_argparser.py b/tests/test_argparser.py index 079e763a..c2de5d83 100644 --- a/tests/test_argparser.py +++ b/tests/test_argparser.py @@ -15,6 +15,7 @@ # You should have received a copy of the GNU General Public License # along with this program. If not, see . +import tempfile import unittest from multiprocessing import cpu_count from pathlib import Path @@ -134,9 +135,9 @@ def test_parse_min_cpu(self): self.assertEqual(parsed_args.n_jobs, cpu_count() // 2) def test_parse_root(self): - parsed_args = parse_args(self.terminal, ["--root", "foo"]) - - self.assertEqual(parsed_args.root, Path("foo")) + with tempfile.TemporaryDirectory() as tmpdir: + parsed_args = parse_args(self.terminal, ["--root", tmpdir]) + self.assertEqual(parsed_args.root, Path(tmpdir)) def test_parse_fix(self): parsed_args = parse_args(self.terminal, ["--fix"]) @@ -149,6 +150,8 @@ def test_parse_ignore_warnings(self): self.assertTrue(parsed_args.ignore_warnings) def test_parse_log_file_statistic(self): - parsed_args = parse_args(self.terminal, ["--log-file-statistic", "foo"]) - - self.assertEqual(parsed_args.log_file_statistic, Path("foo")) + with tempfile.NamedTemporaryFile() as tmpfile: + parsed_args = parse_args( + self.terminal, ["--log-file-statistic", str(tmpfile.name)] + ) + self.assertEqual(parsed_args.log_file_statistic, Path(tmpfile.name)) diff --git a/tests/test_runner.py b/tests/test_runner.py index b13b2cfa..2779cf67 100644 --- a/tests/test_runner.py +++ b/tests/test_runner.py @@ -39,6 +39,8 @@ _here = Path(__file__).parent +DEFAULT_CONFIG = {} + class TestRunner(unittest.TestCase): def setUp(self): @@ -51,6 +53,7 @@ def test_runner_with_all_plugins(self): n_jobs=1, reporter=self._reporter, root=self.root, + plugins_config=DEFAULT_CONFIG, ) plugins = _FILE_PLUGINS + _FILES_PLUGINS @@ -73,6 +76,7 @@ def test_runner_with_excluded_plugins(self): reporter=self._reporter, excluded_plugins=excluded_plugins, root=self.root, + plugins_config=DEFAULT_CONFIG, ) for plugin in runner.plugins: @@ -88,6 +92,7 @@ def test_runner_with_included_plugins(self): reporter=self._reporter, included_plugins=included_plugins, root=self.root, + plugins_config=DEFAULT_CONFIG, ) self.assertEqual(len(runner.plugins), 2) @@ -113,6 +118,7 @@ def test_runner_run_ok(self): reporter=self._reporter, included_plugins=included_plugins, root=self.root, + plugins_config=DEFAULT_CONFIG, ) with redirect_stdout(io.StringIO()) as _: sys_exit = runner.run([nasl_file]) @@ -156,6 +162,7 @@ def test_runner_run_error(self): reporter=self._reporter, included_plugins=included_plugins, root=self.root, + plugins_config=DEFAULT_CONFIG, ) with redirect_stdout(io.StringIO()) as _: @@ -204,6 +211,7 @@ def test_runner_run_fail_with_verbose_level_2(self): reporter=reporter, included_plugins=[CheckScriptVersionAndLastModificationTags.name], root=self.root, + plugins_config=DEFAULT_CONFIG, ) with redirect_stdout(io.StringIO()) as f: @@ -250,6 +258,7 @@ def test_runner_run_ok_with_verbose_level_3(self): reporter=reporter, included_plugins=included_plugins, root=self.root, + plugins_config=DEFAULT_CONFIG, ) with redirect_stdout(io.StringIO()) as f: @@ -286,6 +295,7 @@ def test_runner_run_ok_with_verbose_level_2(self): reporter=reporter, included_plugins=included_plugins, root=self.root, + plugins_config=DEFAULT_CONFIG, ) with redirect_stdout(io.StringIO()) as f: @@ -325,6 +335,7 @@ def test_runner_run_ok_with_verbose_level_1(self): n_jobs=1, included_plugins=included_plugins, root=self.root, + plugins_config=DEFAULT_CONFIG, ) with redirect_stdout(io.StringIO()) as f: @@ -348,6 +359,7 @@ def test_no_plugins(self): reporter=self._reporter, included_plugins=["foo"], root=self.root, + plugins_config=DEFAULT_CONFIG, ) nasl_file = _here / "plugins" / "test.nasl" @@ -380,6 +392,7 @@ def test_runner_log_file(self): n_jobs=1, included_plugins=included_plugins, root=self.root, + plugins_config=DEFAULT_CONFIG, ) with redirect_stdout(io.StringIO()): runner.run([nasl_file]) @@ -427,6 +440,7 @@ def test_runner_log_file_fail(self): reporter=reporter, included_plugins=included_plugins, root=self.root, + plugins_config=DEFAULT_CONFIG, ) with redirect_stdout(io.StringIO()): runner.run([nasl_file]) @@ -464,6 +478,7 @@ def test_runner_run_ok_with_ignore_warnings(self): included_plugins=included_plugins, root=self.root, ignore_warnings=True, + plugins_config=DEFAULT_CONFIG, ) with redirect_stdout(io.StringIO()) as f: @@ -489,6 +504,7 @@ def test_runner_run_fail_without_ignore_warnings(self): reporter=reporter, included_plugins=included_plugins, root=self.root, + plugins_config=DEFAULT_CONFIG, ) with redirect_stdout(io.StringIO()) as f: @@ -529,6 +545,7 @@ def test_runner_log_file_statistic(self): n_jobs=1, included_plugins=included_plugins, root=self.root, + plugins_config=DEFAULT_CONFIG, ) with redirect_stdout(io.StringIO()): runner.run([nasl_file]) @@ -575,6 +592,7 @@ def test_runner_fail_log_file_statistic(self): n_jobs=1, included_plugins=included_plugins, root=self.root, + plugins_config=DEFAULT_CONFIG, ) with redirect_stdout(io.StringIO()): runner.run([nasl_file]) diff --git a/troubadix.toml b/troubadix.toml new file mode 100644 index 00000000..127adf94 --- /dev/null +++ b/troubadix.toml @@ -0,0 +1,58 @@ +# Use Taplo / VS Code Even Better TOML for formatting + +# File Structure: +# [plugin_name] +# config_a = value_a +# config_b = value_b + +[check_http_links_in_tags] +# Strings that should be ignored because they contain a valid URL type in a tag +exclusions = [ + "The payloads try to open a connection to www.google.com", + "The script attempts to connect to www.google.com", + "to retrieve a web page from www.google.com", + "Terms of use at https://www.verisign.com/rpa", + "Subject: commonName=www.paypal.com", + "example.com", + "example.org", + "www.exam", + "sampling the resolution of a name (www.google.com)", + "once with 'www.' and once without", + "wget http://www.javaop.com/~ron/tmp/nc", + "as www.windowsupdate.com. (BZ#506016)", + "located at http://sambarserver/session/pagecount.", + "http://rest.modx.com", + "ftp:// ", + "ftp://'", + "ftp://)", + "ftp.c", + "ftp.exe", + "using special ftp://", + "running ftp.", + "ftp. The vulnerability", + "'http://' protocol", + "handle properly", + "Switch to git+https://", + "wget https://compromised-domain.com/important-file", + "the https:// scheme", + "https://www.phishingtarget.com@evil.com", + "distributions on ftp.proftpd.org have all been", + "information from www.mutt.org:", + "According to www.tcpdump.org:", + "According to www.kde.org:", + "From the www.info-zip.org site:", + " (www.isg.rhul.ac.uk) for discovering this flaw and Adam Langley and", + "Sorry about having to reissue this one -- I pulled it from ftp.gnu.org not", + # e.g.: + # Since gedit supports opening files via 'http://' URLs + "'http://'", + "'https://'", + "http://internal-host$1 is still insecure", + "http:// ", + "https:// ", + "such as 'http://:80'", + "", + "https://username:password@proxy:8080", + "sun.net.www.http.KeepAliveCache", + "www.foo.com", +] diff --git a/troubadix/argparser.py b/troubadix/argparser.py index d4d08e6c..27f744cf 100644 --- a/troubadix/argparser.py +++ b/troubadix/argparser.py @@ -28,12 +28,17 @@ def directory_type(string: str) -> Path: directory_path = Path(string) - if directory_path.exists() and not directory_path.is_dir(): + if not directory_path.is_dir(): raise ValueError(f"{string} is not a directory.") return directory_path def file_type(string: str) -> Path: + """if statement is correct and should not be changed + checks: + - is path an existing file -> file can be used + - is a non-existent path -> file can be created at that location later + """ file_path = Path(string) if file_path.exists() and not file_path.is_file(): raise ValueError(f"{string} is not a file.") @@ -230,6 +235,17 @@ def parse_args( help="Don't print the statistic", ) + parser.add_argument( + "-c", + "--config", + type=Path, + default="troubadix.toml", + help=( + "Specify the path to the file that contains additional " + "configuration for the plugins" + ), + ) + if not args: print("No arguments given.", file=sys.stderr) parser.print_help(sys.stdout) diff --git a/troubadix/plugin.py b/troubadix/plugin.py index f04d7350..9e20984c 100644 --- a/troubadix/plugin.py +++ b/troubadix/plugin.py @@ -79,12 +79,65 @@ def __init__(self, *, root: Path, nasl_files: Iterable[Path]) -> None: self.nasl_files = nasl_files +class ConfigurationError(Exception): + """Custom exception for plugin_configuration errors.""" + + class Plugin(ABC): """A linter plugin""" name: str = None description: str = None + # Value to indicate that a plugin depends on an external configuration + require_external_config = False + + def __init__(self, config: dict) -> None: + if self.require_external_config: + self.config = self.extract_plugin_config(config) + + def extract_plugin_config(self, config: dict) -> dict: + """ + extracts the configuration for a specific plugin + from the entire configuration. + + Args: + config (dict): The entire configuration dictionary. + + Returns: + dict: The configuration dictionary for the specific plugin. + + Raises: + ConfigurationError: If no configuration exists or validation fails. + """ + if self.name not in config: + raise ConfigurationError( + f"Configuration for plugin '{self.name}' is missing." + ) + plugin_config = config[self.name] + self.validate_plugin_config(plugin_config) + return plugin_config + + def validate_plugin_config(self, config: dict) -> None: + """ + Validates the configuration for a specific plugin + + Not @abstract due to only being necessary + if require_external_config is true + + Args: + config (dict): The configuration dictionary for the specific plugin. + + Raises: + ConfigurationError: If the plugins required keys are missing. + """ + raise RuntimeError( + f"{self.__class__.__name__} has not implemented method" + " 'validate_and_extract_plugin_config'." + " This method should be overridden in subclasses," + " if they require external config" + ) + @abstractmethod def run(self) -> Iterator[LinterResult]: pass @@ -96,14 +149,19 @@ def fix(self) -> Iterator[LinterResult]: class FilesPlugin(Plugin): """A plugin that does checks over all files""" - def __init__(self, context: FilesPluginContext) -> None: + def __init__(self, context: FilesPluginContext, **kwargs) -> None: + if "config" in kwargs: + super().__init__(kwargs["config"]) self.context = context class FilePlugin(Plugin): """A plugin that does checks on single files""" - def __init__(self, context: FilePluginContext) -> None: + def __init__(self, context: FilePluginContext, **kwargs) -> None: + if "config" in kwargs: + super().__init__(kwargs["config"]) + self.context = context diff --git a/troubadix/plugins/http_links_in_tags.py b/troubadix/plugins/http_links_in_tags.py index eed0a778..5734bc05 100644 --- a/troubadix/plugins/http_links_in_tags.py +++ b/troubadix/plugins/http_links_in_tags.py @@ -17,15 +17,29 @@ import re from itertools import chain -from typing import AnyStr, Iterator +from typing import Iterator from troubadix.helper import SpecialScriptTag, get_common_tag_patterns from troubadix.helper.patterns import get_special_script_tag_pattern -from troubadix.plugin import FilePlugin, LinterError, LinterResult +from troubadix.plugin import ( + ConfigurationError, + FilePlugin, + LinterError, + LinterResult, +) class CheckHttpLinksInTags(FilePlugin): name = "check_http_links_in_tags" + require_external_config = True + + def validate_plugin_config(self, config: dict) -> None: + """Check the plugin configuration for keys required by this plugin""" + if "exclusions" not in config: + raise ConfigurationError( + f"Configuration for plugin '{self.name}' is missing " + "required key: 'exclusions'" + ) def run(self) -> Iterator[LinterResult]: if self.context.nasl_file.suffix == ".inc": @@ -49,6 +63,11 @@ def contains_http_link_in_tag(self) -> Iterator[LinterResult]: nasl_file: The VT that is going to be checked file_content: The content of the file that is going to be checked + config: The plugin configuration provided + by the plugin_configuration.toml file. + config must include keys: + exclusions: A list of Strings that should be ignored + due to containing a valid use of a url in a tag """ file_content = self.context.file_content @@ -64,7 +83,9 @@ def contains_http_link_in_tag(self) -> Iterator[LinterResult]: if http_link_matches: for http_link_match in http_link_matches: if http_link_match: - if self.check_to_continue(http_link_match.group(0)): + if self.check_to_continue( + http_link_match.group(0), + ): continue yield LinterError( @@ -116,61 +137,8 @@ def contains_nvd_mitre_link_in_xref(self) -> Iterator[LinterResult]: plugin=self.name, ) - @staticmethod - def check_to_continue(http_link_match_group: AnyStr) -> bool: - # When adding new entries to this list, please also add a testcase to - # tests/plugins/test_http_links_in_tags.py -> test_http_link_in_tags_ok - exclusions = [ - "The payloads try to open a connection to www.google.com", - "The script attempts to connect to www.google.com", - "to retrieve a web page from www.google.com", - "Terms of use at https://www.verisign.com/rpa", - "Subject: commonName=www.paypal.com", - "example.com", - "example.org", - "www.exam", - "sampling the resolution of a name (www.google.com)", - "once with 'www.' and once without", - "wget http://www.javaop.com/~ron/tmp/nc", - "as www.windowsupdate.com. (BZ#506016)", - "located at http://sambarserver/session/pagecount.", - "http://rest.modx.com", - "ftp:// ", - "ftp://'", - "ftp://)", - "ftp.c", - "ftp.exe", - "using special ftp://", - "running ftp.", - "ftp. The vulnerability", - "'http://' protocol", - "handle properly", - "Switch to git+https://", - "wget https://compromised-domain.com/important-file", - "the https:// scheme", - "https://www.phishingtarget.com@evil.com", - "distributions on ftp.proftpd.org have all been", - "information from www.mutt.org:", - "According to www.tcpdump.org:", - "According to www.kde.org:", - "From the www.info-zip.org site:", - # pylint: disable=line-too-long - " (www.isg.rhul.ac.uk) for discovering this flaw and Adam Langley and", - "Sorry about having to reissue this one -- I pulled it from ftp.gnu.org not", - # e.g.: - # Since gedit supports opening files via 'http://' URLs - "'http://'", - "'https://'", - "http://internal-host$1 is still insecure", - "http:// ", - "https:// ", - "such as 'http://:80'", - "", - "https://username:password@proxy:8080", - "sun.net.www.http.KeepAliveCache", - "www.foo.com", - ] - + def check_to_continue(self, http_link_match_group: str) -> bool: + exclusions = self.config["exclusions"] return any( exclusion in http_link_match_group for exclusion in exclusions ) diff --git a/troubadix/runner.py b/troubadix/runner.py index e95ada1c..e63bef4a 100644 --- a/troubadix/runner.py +++ b/troubadix/runner.py @@ -17,6 +17,7 @@ import datetime import signal +import sys from multiprocessing import Pool from pathlib import Path from typing import Iterable @@ -30,6 +31,11 @@ from troubadix.reporter import Reporter from troubadix.results import FileResults, Results +try: + import tomllib +except ImportError: + import tomli as tomllib + CHUNKSIZE = 1 # default 1 @@ -53,13 +59,28 @@ def __init__( included_plugins: Iterable[str] = None, fix: bool = False, ignore_warnings: bool = False, - ) -> bool: + plugins_config_path: Path, + ) -> None: # plugins initialization self.plugins = StandardPlugins(excluded_plugins, included_plugins) self._excluded_plugins = excluded_plugins self._included_plugins = included_plugins + self.requires_config = self._check_requires_config() + if self.requires_config: + + # Get the plugins configurations from the external toml file + try: + with open(plugins_config_path, "rb") as file: + self.plugins_config = tomllib.load(file) + except FileNotFoundError: + print(f"Config file '{plugins_config_path}' does not exist") + sys.exit(1) + except tomllib.TOMLDecodeError as e: + print(f"Error decoding TOML file '{plugins_config_path}': {e}") + sys.exit(1) + self._reporter = reporter self._n_jobs = n_jobs self._root = root @@ -90,13 +111,30 @@ def _check_file(self, file_path: Path) -> FileResults: root=self._root, nasl_file=file_path.resolve() ) - for plugin_class in self.plugins.file_plugins: - plugin = plugin_class(context) - + file_plugins = self._initialize_plugins( + context, self.plugins.file_plugins + ) + for plugin in file_plugins: self._check(plugin, results) return results + def _initialize_plugins(self, context, plugin_classes): + return [ + ( + plugin_class(context, config=self.plugins_config) + if plugin_class.require_external_config + else plugin_class(context) + ) + for plugin_class in plugin_classes + ] + + def _check_requires_config(self): + return any( + plugin.require_external_config + for plugin in self.plugins.files_plugins + self.plugins.file_plugins + ) + def _run_pooled(self, files: Iterable[Path]): """Run all plugins that check single files""" self._reporter.set_files_count(len(files)) @@ -104,10 +142,9 @@ def _run_pooled(self, files: Iterable[Path]): try: # run files plugins context = FilesPluginContext(root=self._root, nasl_files=files) - files_plugins = [ - plugin_class(context) - for plugin_class in self.plugins.files_plugins - ] + files_plugins = self._initialize_plugins( + context, self.plugins.files_plugins + ) for results in pool.imap_unordered( self._check_files, files_plugins, chunksize=CHUNKSIZE diff --git a/troubadix/troubadix.py b/troubadix/troubadix.py index 1c683601..a6a6cd09 100644 --- a/troubadix/troubadix.py +++ b/troubadix/troubadix.py @@ -182,6 +182,7 @@ def main(args=None): fix=parsed_args.fix, ignore_warnings=parsed_args.ignore_warnings, root=root, + plugins_config_path=parsed_args.config, ) term.info(f"Start linting {len(files)} files ... ")