From 2de7519a9f300909c28f77ca92cfe6176883009f Mon Sep 17 00:00:00 2001 From: James McLean Date: Mon, 23 Oct 2017 18:08:03 +1030 Subject: [PATCH 1/9] Added prefix and suffix arguments, beginnings of prefix implementation; added verbose argument and updated host scanner to show hosts scanned in verbose mode --- VHostScan.py | 7 ++++++- lib/core/virtual_host_scanner.py | 4 ++++ lib/helpers/wordlist_helper.py | 11 ++++++++++- lib/input.py | 15 +++++++++++++++ 4 files changed, 35 insertions(+), 2 deletions(-) mode change 100644 => 100755 VHostScan.py diff --git a/VHostScan.py b/VHostScan.py old mode 100644 new mode 100755 index 08aecd0..60322be --- a/VHostScan.py +++ b/VHostScan.py @@ -33,7 +33,7 @@ def main(): wordlist_helper = WordList() wordlist, wordlist_types = wordlist_helper.get_wordlist( - arguments.wordlists) + arguments.wordlists, arguments.prefix, arguments.suffix) if len(wordlist) == 0: print("[!] No words found in provided wordlists, unable to scan.") @@ -82,11 +82,16 @@ def main(): wordlist.append(str(ip)) wordlist.append(host) wordlist.extend(aliases) + if arguments.verbose: + print("[!] Discovered {host}/{ip}. Adding...".format(ip=str(ip), host=host)) except (dns.resolver.NXDOMAIN): print("[!] Couldn't find any records (NXDOMAIN)") except (dns.resolver.NoAnswer): print("[!] Couldn't find any records (NoAnswer)") + if arguments.verbose: + print("[>] Scanning with %s items in wordlist" % len(wordlist)) + scanner_args = vars(arguments) scanner_args.update({ 'target': arguments.target_hosts, diff --git a/lib/core/virtual_host_scanner.py b/lib/core/virtual_host_scanner.py index 8dca5ed..79c55d4 100644 --- a/lib/core/virtual_host_scanner.py +++ b/lib/core/virtual_host_scanner.py @@ -59,6 +59,7 @@ def __init__(self, target, wordlist, **kwargs): self.unique_depth = int(kwargs.get('unique_depth', 1)) self.ignore_http_codes = kwargs.get('ignore_http_codes', '404') self.first_hit = kwargs.get('first_hit') + self.verbose = kwargs.get('verbose') self.ignore_content_length = int( kwargs.get('ignore_content_length', 0) @@ -104,6 +105,9 @@ def scan(self): for virtual_host in self.wordlist: hostname = virtual_host.replace('%s', self.base_host) + if self.verbose: + print("[*] Scanning {hostname}".format(hostname=hostname)) + if self.real_port == 80: host_header = hostname else: diff --git a/lib/helpers/wordlist_helper.py b/lib/helpers/wordlist_helper.py index 03791b6..162f9cb 100644 --- a/lib/helpers/wordlist_helper.py +++ b/lib/helpers/wordlist_helper.py @@ -19,7 +19,7 @@ def get_stdin_wordlist(self): return list(line for line in sys.stdin.read().splitlines()) \ if not sys.stdin.isatty() else [] - def get_wordlist(self, wordlist_files=None): + def get_wordlist(self, wordlist_files=None, wordlist_prefix=False, wordlist_suffix=False): default_wordlist_file = DEFAULT_WORDLIST_FILE stdin_words = self.get_stdin_wordlist() @@ -29,11 +29,20 @@ def get_wordlist(self, wordlist_files=None): combined_files = wordlist_files or default_wordlist_file combined = get_combined_word_lists(combined_files) + if combined: words_type = 'wordlists: {}'.format( ', '.join(combined['file_paths'])) self.set_words(words_type=words_type, words=combined['words']) + # Apply prefixes + if wordlist_prefix: + prefixed = [wordlist_prefix + word for word in self.wordlist] + self.wordlist = self.wordlist + prefixed + + #if wordlist_suffix: + + return self.wordlist, self.wordlist_types def set_words(self, words_type, words): diff --git a/lib/input.py b/lib/input.py index ae76914..0468668 100644 --- a/lib/input.py +++ b/lib/input.py @@ -34,6 +34,16 @@ def setup_parser(): help='Set the port to use (default 80).' ) + parser.add_argument( + '--prefix', dest='prefix', default=False, + help='Add a prefix to each item in the word list (dev, test etc)' + ) + + parser.add_argument( + '--suffix', dest='suffix', default=False, + help='Add a suffix to each item in the word list' + ) + parser.add_argument( '-r', dest='real_port', type=int, default=False, help='The real port of the webserver to use in headers when ' @@ -98,6 +108,11 @@ def setup_parser(): help='If set then simple WAF bypass headers will be sent.' ) + parser.add_argument( + '-v', dest='verbose', action='store_true', default=False, + help='Print verbose output' + ) + output = parser.add_mutually_exclusive_group() output.add_argument( '-oN', dest='output_normal', From ed4a3aa5b5e8d6be5361e7a3528027c2030b0db8 Mon Sep 17 00:00:00 2001 From: James McLean Date: Mon, 23 Oct 2017 19:23:19 +1030 Subject: [PATCH 2/9] Reworked wordlist building for prefix and suffix code, added IP address validation --- lib/helpers/wordlist_helper.py | 39 ++++++++++++++++++++++++++++++---- 1 file changed, 35 insertions(+), 4 deletions(-) diff --git a/lib/helpers/wordlist_helper.py b/lib/helpers/wordlist_helper.py index 162f9cb..e564dd2 100644 --- a/lib/helpers/wordlist_helper.py +++ b/lib/helpers/wordlist_helper.py @@ -37,14 +37,45 @@ def get_wordlist(self, wordlist_files=None, wordlist_prefix=False, wordlist_suff # Apply prefixes if wordlist_prefix: - prefixed = [wordlist_prefix + word for word in self.wordlist] - self.wordlist = self.wordlist + prefixed + prefixed = [] + for word in self.wordlist: + if(word == '%s'): + continue + elif(self.valid_ip(word)): + continue + else: + prefixed.append(wordlist_prefix + word) - #if wordlist_suffix: - + if len(prefixed) > 0: + self.wordlist = self.wordlist + prefixed + if wordlist_suffix: + suffixed = [] + for word in self.wordlist: + if(word == '%s'): + continue + elif(self.valid_ip(word)): + continue + elif(".%s" in word): + split = word.split(".") + suffixed.append(split[0] + wordlist_suffix + ".%s") + else: + suffixed.append(word + wordlist_suffix) + + if len(suffixed) > 0: + self.wordlist = self.wordlist + suffixed + return self.wordlist, self.wordlist_types def set_words(self, words_type, words): self.wordlist_types.append(words_type) self.wordlist.extend(words) + + def valid_ip(self, address): + try: + host_bytes = address.split('.') + valid = [int(b) for b in host_bytes] + valid = [b for b in valid if b >= 0 and b<=255] + return len(host_bytes) == 4 and len(valid) == 4 + except: + return False From 517d69cc435eda5d9d1f2f375626d8883b33fed3 Mon Sep 17 00:00:00 2001 From: James McLean Date: Mon, 23 Oct 2017 19:51:08 +1030 Subject: [PATCH 3/9] Added defaults for prefix, suffix and verbose to tests, fixed broken import --- tests/helpers/test_wordlist_helper.py | 2 +- tests/test_input.py | 9 +++++++++ 2 files changed, 10 insertions(+), 1 deletion(-) diff --git a/tests/helpers/test_wordlist_helper.py b/tests/helpers/test_wordlist_helper.py index 648575c..2abecff 100644 --- a/tests/helpers/test_wordlist_helper.py +++ b/tests/helpers/test_wordlist_helper.py @@ -1,6 +1,6 @@ import unittest import pytest -from mock import patch +from unittest.mock import patch from lib.helpers.wordlist_helper import WordList from lib.helpers.wordlist_helper import DEFAULT_WORDLIST_FILE diff --git a/tests/test_input.py b/tests/test_input.py index 011765e..d69bd5a 100644 --- a/tests/test_input.py +++ b/tests/test_input.py @@ -32,6 +32,9 @@ def test_parse_arguments_default_value(tmpdir): 'output_json': None, 'output_grepable' : None, 'ssl': False, + 'prefix': False, + 'suffix': False, + 'verbose': False } assert vars(arguments) == expected_arguments @@ -59,6 +62,9 @@ def test_parse_arguments_custom_arguments(tmpdir): '--user-agent', 'some-user-agent', '--waf', '-oN', '/tmp/on', + '--prefix','dev-', + '--suffix','test', + '-v' ] arguments = cli_argument_parser().parse(argv) @@ -83,6 +89,9 @@ def test_parse_arguments_custom_arguments(tmpdir): 'output_normal': '/tmp/on', 'output_json': None, 'output_grepable' : None, + 'prefix': 'dev-', + 'suffix': 'test', + 'verbose': True } assert vars(arguments) == expected_arguments From 484e21d6f9c68f841ab5cbfdf4ed2f0e1a31dcb6 Mon Sep 17 00:00:00 2001 From: James McLean Date: Mon, 23 Oct 2017 20:13:29 +1030 Subject: [PATCH 4/9] Added tests for prefix and suffix on plain words --- tests/helpers/test_wordlist_helper.py | 33 +++++++++++++++++++++++++++ 1 file changed, 33 insertions(+) diff --git a/tests/helpers/test_wordlist_helper.py b/tests/helpers/test_wordlist_helper.py index 2abecff..5aa4e79 100644 --- a/tests/helpers/test_wordlist_helper.py +++ b/tests/helpers/test_wordlist_helper.py @@ -18,6 +18,7 @@ def user_wordlist(request, tmpdir_factory): @pytest.mark.usefixtures('user_wordlist') class TestWordList(unittest.TestCase): + def setUp(self): self.wordlist = WordList() with open(DEFAULT_WORDLIST_FILE, 'r') as word_file: @@ -45,3 +46,35 @@ def test_using_default_wordlist(self): with patch('lib.helpers.wordlist_helper.WordList.get_stdin_wordlist', return_value=stdin_wordlist): wordlist, wordlist_types = self.wordlist.get_wordlist() self.assertEqual(wordlist, self.default_wordlist) + + def test_ip_using_prefix(self): + stdin_wordlist = ['127.0.0.1'] + prefix = 'dev-' + with patch('lib.helpers.wordlist_helper.WordList.get_stdin_wordlist', return_value=stdin_wordlist): + wordlist, wordlist_types = self.wordlist.get_wordlist(None, prefix) + self.assertEqual(wordlist, stdin_wordlist) + + def test_ip_using_suffix(self): + stdin_wordlist = ['127.0.0.1'] + suffix = 'test' + with patch('lib.helpers.wordlist_helper.WordList.get_stdin_wordlist', return_value=stdin_wordlist): + wordlist, wordlist_types = self.wordlist.get_wordlist(None,None,suffix) + self.assertEqual(wordlist,stdin_wordlist) + + def test_word_with_prefix(self): + stdin_wordlist = ['www','www2','www3'] + expected_wordlist = stdin_wordlist + ['dev-www','dev-www2','dev-www3'] + prefix = 'dev-' + with patch('lib.helpers.wordlist_helper.WordList.get_stdin_wordlist', return_value=stdin_wordlist): + wordlist, wordlist_types = self.wordlist.get_wordlist(None,prefix) + self.assertEqual(wordlist,expected_wordlist) + + def test_words_with_suffix(self): + stdin_wordlist = ['www','www2','www3'] + expected_wordlist = stdin_wordlist + ['wwwtest','www2test','www3test'] + suffix = 'test' + with patch('lib.helpers.wordlist_helper.WordList.get_stdin_wordlist', return_value=stdin_wordlist): + wordlist, wordlist_types = self.wordlist.get_wordlist(None,None,suffix) + self.assertEqual(wordlist, expected_wordlist) + + From 4fd47912d330519c0f40a2dc93831f1364794e99 Mon Sep 17 00:00:00 2001 From: James McLean Date: Mon, 23 Oct 2017 20:20:25 +1030 Subject: [PATCH 5/9] Added more tests for prefix and suffix --- tests/helpers/test_wordlist_helper.py | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+) diff --git a/tests/helpers/test_wordlist_helper.py b/tests/helpers/test_wordlist_helper.py index 5aa4e79..a5b3d54 100644 --- a/tests/helpers/test_wordlist_helper.py +++ b/tests/helpers/test_wordlist_helper.py @@ -77,4 +77,22 @@ def test_words_with_suffix(self): wordlist, wordlist_types = self.wordlist.get_wordlist(None,None,suffix) self.assertEqual(wordlist, expected_wordlist) + def test_words_with_host_and_prefix(self): + stdin_wordlist = ['www.%s'] + expected_wordlist = stdin_wordlist + ['test-www.%s'] + prefix = 'test-' + with patch('lib.helpers.wordlist_helper.WordList.get_stdin_wordlist', return_value=stdin_wordlist): + wordlist, wordlist_types = self.wordlist.get_wordlist(None, prefix) + self.assertEqual(wordlist, expected_wordlist) + + def test_words_with_host_and_suffix(self): + stdin_wordlist = ['www.%s'] + expected_wordlist = stdin_wordlist + ['wwwtest.%s'] + suffix = 'test' + with patch('lib.helpers.wordlist_helper.WordList.get_stdin_wordlist', return_value=stdin_wordlist): + wordlist, wordlist_types = self.wordlist.get_wordlist(None,None,suffix) + self.assertEqual(wordlist, expected_wordlist) + + + From 8cb75584a5180e3f9e4e0e233e74676e5f5fc860 Mon Sep 17 00:00:00 2001 From: James McLean Date: Mon, 23 Oct 2017 20:25:34 +1030 Subject: [PATCH 6/9] Updated README.md with new command line args --- README.md | 3 +++ 1 file changed, 3 insertions(+) diff --git a/README.md b/README.md index d666135..d93eb07 100644 --- a/README.md +++ b/README.md @@ -40,6 +40,8 @@ $ pip install -r requirements.txt | -r REAL_PORT | The real port of the webserver to use in headers when not 80 (see RFC2616 14.23), useful when pivoting through ssh/nc etc (default to PORT). | | --ignore-http-codes IGNORE_HTTP_CODES | Comma separated list of http codes to ignore with virtual host scans (default 404). | | --ignore-content-length IGNORE_CONTENT_LENGTH | Ignore content lengths of specificed amount. | +| --prefix PREFIX | Add a prefix to each item in the wordlist, to add dev-, test- etc | +| --suffix SUFFIX | Add a suffix to each item in the wordlist, to add dev, dev | | --first-hit | Return first successful result. Only use in scenarios where you are sure no catch-all is configured (such as a CTF). | | --unique-depth UNIQUE_DEPTH | Show likely matches of page content that is found x times (default 1). | | --ssl | If set then connections will be made over HTTPS instead of HTTP. | @@ -52,6 +54,7 @@ $ pip install -r requirements.txt | -oN OUTPUT_NORMAL | Normal output printed to a file when the -oN option is specified with a filename argument. | | -oG OUTPUT_GREPABLE | Grepable output printed to a file when the -oG is specified with a filename argument. | | -oJ OUTPUT_JSON | JSON output printed to a file when the -oJ option is specified with a filename argument. | +| -v VERBOSE | Increase the output of the tool to show progress | ## Usage Examples From 38756ec568bd102b0a5f06580b7c98b260686491 Mon Sep 17 00:00:00 2001 From: James McLean Date: Mon, 23 Oct 2017 20:27:19 +1030 Subject: [PATCH 7/9] Updated README.md with new command line args --- README.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index d93eb07..ad58587 100644 --- a/README.md +++ b/README.md @@ -40,8 +40,8 @@ $ pip install -r requirements.txt | -r REAL_PORT | The real port of the webserver to use in headers when not 80 (see RFC2616 14.23), useful when pivoting through ssh/nc etc (default to PORT). | | --ignore-http-codes IGNORE_HTTP_CODES | Comma separated list of http codes to ignore with virtual host scans (default 404). | | --ignore-content-length IGNORE_CONTENT_LENGTH | Ignore content lengths of specificed amount. | -| --prefix PREFIX | Add a prefix to each item in the wordlist, to add dev-, test- etc | -| --suffix SUFFIX | Add a suffix to each item in the wordlist, to add dev, dev | +| --prefix PREFIX | Add a prefix to each item in the wordlist, to add dev-\, test-\ etc | +| --suffix SUFFIX | Add a suffix to each item in the wordlist, to add \dev, \dev | | --first-hit | Return first successful result. Only use in scenarios where you are sure no catch-all is configured (such as a CTF). | | --unique-depth UNIQUE_DEPTH | Show likely matches of page content that is found x times (default 1). | | --ssl | If set then connections will be made over HTTPS instead of HTTP. | From 78a2ced158c5dbcd0c8997ecf3e815de70908897 Mon Sep 17 00:00:00 2001 From: James McLean Date: Mon, 23 Oct 2017 21:35:00 +1030 Subject: [PATCH 8/9] Fixed pep8 issues --- VHostScan.py | 3 ++- lib/helpers/wordlist_helper.py | 12 ++++++++---- 2 files changed, 10 insertions(+), 5 deletions(-) diff --git a/VHostScan.py b/VHostScan.py index 60322be..723efc9 100755 --- a/VHostScan.py +++ b/VHostScan.py @@ -83,7 +83,8 @@ def main(): wordlist.append(host) wordlist.extend(aliases) if arguments.verbose: - print("[!] Discovered {host}/{ip}. Adding...".format(ip=str(ip), host=host)) + print("[!] Discovered {host}/{ip}. Adding...". + format(ip=str(ip), host=host)) except (dns.resolver.NXDOMAIN): print("[!] Couldn't find any records (NXDOMAIN)") except (dns.resolver.NoAnswer): diff --git a/lib/helpers/wordlist_helper.py b/lib/helpers/wordlist_helper.py index e564dd2..5e0e3d3 100644 --- a/lib/helpers/wordlist_helper.py +++ b/lib/helpers/wordlist_helper.py @@ -19,7 +19,11 @@ def get_stdin_wordlist(self): return list(line for line in sys.stdin.read().splitlines()) \ if not sys.stdin.isatty() else [] - def get_wordlist(self, wordlist_files=None, wordlist_prefix=False, wordlist_suffix=False): + def get_wordlist(self, + wordlist_files=None, + wordlist_prefix=False, + wordlist_suffix=False): + default_wordlist_file = DEFAULT_WORDLIST_FILE stdin_words = self.get_stdin_wordlist() @@ -63,8 +67,8 @@ def get_wordlist(self, wordlist_files=None, wordlist_prefix=False, wordlist_suff suffixed.append(word + wordlist_suffix) if len(suffixed) > 0: - self.wordlist = self.wordlist + suffixed - + self.wordlist = self.wordlist + suffixed + return self.wordlist, self.wordlist_types def set_words(self, words_type, words): @@ -75,7 +79,7 @@ def valid_ip(self, address): try: host_bytes = address.split('.') valid = [int(b) for b in host_bytes] - valid = [b for b in valid if b >= 0 and b<=255] + valid = [b for b in valid if b >= 0 and b <= 255] return len(host_bytes) == 4 and len(valid) == 4 except: return False From e920824c3b8062c191b0250c1a732a43e6d87de6 Mon Sep 17 00:00:00 2001 From: Michael <886344+codingo@users.noreply.github.com> Date: Tue, 24 Oct 2017 08:53:35 +1000 Subject: [PATCH 9/9] Update __version__.py --- lib/core/__version__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/core/__version__.py b/lib/core/__version__.py index 026177a..ea65b99 100644 --- a/lib/core/__version__.py +++ b/lib/core/__version__.py @@ -2,4 +2,4 @@ # |V|H|o|s|t|S|c|a|n| Developed by @codingo_ & @__timk # +-+-+-+-+-+-+-+-+-+ https://github.com/codingo/VHostScan -__version__ = '1.7.1' +__version__ = '1.8'