From f78e375764eb3a46aaa10dd86f5e3a1364b6356e Mon Sep 17 00:00:00 2001 From: user Date: Mon, 22 Jul 2024 15:58:51 +0200 Subject: [PATCH 01/10] Add Shodan & Hackertarget Engines --- requirements.txt | 2 ++ sublist3r.py | 91 +++++++++++++++++++++++++++++++++++++++++++++--- 2 files changed, 89 insertions(+), 4 deletions(-) diff --git a/requirements.txt b/requirements.txt index 498ea9d..256072c 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,3 +1,5 @@ argparse dnspython requests +bs4 + diff --git a/sublist3r.py b/sublist3r.py index 760e5ce..fc3b2bc 100755 --- a/sublist3r.py +++ b/sublist3r.py @@ -16,6 +16,7 @@ import socket import json from collections import Counter +from bs4 import BeautifulSoup # external modules from subbrute import subbrute @@ -79,7 +80,9 @@ def banner(): ___) | |_| | |_) | | \__ \ |_ ___) | | |____/ \__,_|_.__/|_|_|___/\__|____/|_|%s%s - # Coded By Ahmed Aboul-Ela - @aboul3la + # Coded By: + # Ahmed Aboul-Ela - @aboul3la + # fmjal - @fmjal """ % (R, W, Y)) @@ -850,6 +853,84 @@ def extract_domains(self, resp): except Exception as e: pass +class HTEnum(enumratorBaseThreaded): + def __init__(self, domain, subdomains=None, q=None, silent=False, verbose=True): + subdomains = subdomains or [] + base_url = 'https://api.hackertarget.com/hostsearch/?q={domain}' + self.engine_name = "HTEnum" + self.q = q + super(HTEnum, self).__init__(base_url, self.engine_name, domain, subdomains, q=q, silent=silent, verbose=verbose) + return + + def req(self, url): + try: + resp = self.session.get(url, headers=self.headers, timeout=self.timeout) + except Exception as e: + resp = None + + return self.get_response(resp) + + def get_response(self, resp): + if resp and resp.status_code == 200 and not resp.text.startswith("API Count"): + return resp.text + return None + + def extract_domains(self, resp): + for line in resp.splitlines(): + subdomain = line.split(',')[0] + if subdomain and subdomain not in self.subdomains: + self.subdomains.append(subdomain) + + def enumerate(self): + url = self.base_url.format(domain=self.domain) + resp = self.req(url) + if not resp: + return self.subdomains + + self.extract_domains(resp) + return self.subdomains + +class ShodanEnum(enumratorBaseThreaded): + def __init__(self, domain, subdomains=None, q=None, silent=False, verbose=True): + self.domain=domain + subdomains = subdomains or [] + base_url = 'https://www.shodan.io/domain/{domain}' + self.engine_name = "Shodan" + self.q = q + super(ShodanEnum, self).__init__(base_url, self.engine_name, domain, subdomains, q=q, silent=silent, verbose=verbose) + + def req(self, url): + try: + resp = self.session.get(url, headers=self.headers, timeout=self.timeout) + except Exception as e: + resp = None + + return self.get_response(resp) + + def get_response(self, resp): + if resp and resp.status_code == 200: + return resp.text + return None + + def extract_domains(self, resp): + soup = BeautifulSoup(resp, 'html.parser') + subdomains_list = soup.find('ul', id='subdomains') + if subdomains_list: + for li in subdomains_list.find_all('li'): + subdomain = li.get_text().strip() + if subdomain and subdomain not in self.subdomains: + if subdomain != "*" and subdomain != "_dmarc": + self.subdomains.append(f'{subdomain}.{self.domain}') + + def enumerate(self): + url = self.base_url.format(domain=self.domain) + resp = self.req(url) + if not resp: + return self.subdomains + + self.extract_domains(resp) + return self.subdomains + class portscan(): def __init__(self, subdomains, ports): @@ -922,7 +1003,9 @@ def main(domain, threads, savefile, ports, silent, verbose, enable_bruteforce, e 'virustotal': Virustotal, 'threatcrowd': ThreatCrowd, 'ssl': CrtSearch, - 'passivedns': PassiveDNS + 'passivedns': PassiveDNS, + "HTEnum":HTEnum, + "Shodan":ShodanEnum } chosenEnums = [] @@ -930,8 +1013,8 @@ def main(domain, threads, savefile, ports, silent, verbose, enable_bruteforce, e if engines is None: chosenEnums = [ BaiduEnum, YahooEnum, GoogleEnum, BingEnum, AskEnum, - NetcraftEnum, DNSdumpster, Virustotal, ThreatCrowd, - CrtSearch, PassiveDNS + NetcraftEnum, DNSdumpster, ThreatCrowd, + CrtSearch, PassiveDNS,HTEnum,ShodanEnum ] else: engines = engines.split(',') From 9c98126f564b0394b06813c486032d86b4a16e07 Mon Sep 17 00:00:00 2001 From: user Date: Mon, 22 Jul 2024 16:05:26 +0200 Subject: [PATCH 02/10] Fixed version number --- setup.py | 2 +- sublist3r.py | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/setup.py b/setup.py index eb2383c..fc3384e 100644 --- a/setup.py +++ b/setup.py @@ -2,7 +2,7 @@ setup( name='Sublist3r', - version='1.0', + version='1.1', python_requires='>=2.7', install_requires=['dnspython', 'requests', 'argparse; python_version==\'2.7\''], packages=find_packages()+['.'], diff --git a/sublist3r.py b/sublist3r.py index fc3b2bc..607022b 100755 --- a/sublist3r.py +++ b/sublist3r.py @@ -81,8 +81,8 @@ def banner(): |____/ \__,_|_.__/|_|_|___/\__|____/|_|%s%s # Coded By: - # Ahmed Aboul-Ela - @aboul3la - # fmjal - @fmjal + # - Ahmed Aboul-Ela - @aboul3la + # - fmjal - @fmjal """ % (R, W, Y)) From 6a37357854c9320d1d30a321c9de9bc99932b303 Mon Sep 17 00:00:00 2001 From: user Date: Thu, 1 Aug 2024 16:28:31 +0200 Subject: [PATCH 03/10] . --- setup.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/setup.py b/setup.py index fc3384e..f921749 100644 --- a/setup.py +++ b/setup.py @@ -1,8 +1,8 @@ from setuptools import setup, find_packages setup( - name='Sublist3r', - version='1.1', + name='sublist3r', + version='2.0', python_requires='>=2.7', install_requires=['dnspython', 'requests', 'argparse; python_version==\'2.7\''], packages=find_packages()+['.'], From b61f10500673c015b932cbdb7bbc2b99a702cae1 Mon Sep 17 00:00:00 2001 From: fmjal <165020418+fmjal@users.noreply.github.com> Date: Thu, 1 Aug 2024 16:29:45 +0200 Subject: [PATCH 04/10] Update README.md --- README.md | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/README.md b/README.md index c860b29..7e1203c 100644 --- a/README.md +++ b/README.md @@ -1,9 +1,15 @@ -## About Sublist3r +# About Sublist3r Sublist3r is a python tool designed to enumerate subdomains of websites using OSINT. It helps penetration testers and bug hunters collect and gather subdomains for the domain they are targeting. Sublist3r enumerates subdomains using many search engines such as Google, Yahoo, Bing, Baidu and Ask. Sublist3r also enumerates subdomains using Netcraft, Virustotal, ThreatCrowd, DNSdumpster and ReverseDNS. [subbrute](https://github.com/TheRook/subbrute) was integrated with Sublist3r to increase the possibility of finding more subdomains using bruteforce with an improved wordlist. The credit goes to TheRook who is the author of subbrute. +## Key Updates/Changes + +* Added `--silent`,`-s` flag to handle unix-like output piping +* Added Hackertarget & Shodan Enumeraton of Domain +* Removed Defunct Virustotal API + ## Screenshots ![Sublist3r](http://www.secgeek.net/images/Sublist3r.png "Sublist3r in action") From a059384df57add7fbe96713ced0d0c1c9434bc28 Mon Sep 17 00:00:00 2001 From: user Date: Thu, 1 Aug 2024 17:17:32 +0200 Subject: [PATCH 05/10] . --- setup.py | 2 +- sublist3r.py | 11 +++++++---- 2 files changed, 8 insertions(+), 5 deletions(-) mode change 100755 => 100644 sublist3r.py diff --git a/setup.py b/setup.py index f921749..36b86a2 100644 --- a/setup.py +++ b/setup.py @@ -1,7 +1,7 @@ from setuptools import setup, find_packages setup( - name='sublist3r', + name='Sublist3r', version='2.0', python_requires='>=2.7', install_requires=['dnspython', 'requests', 'argparse; python_version==\'2.7\''], diff --git a/sublist3r.py b/sublist3r.py old mode 100755 new mode 100644 index 607022b..f85df4a --- a/sublist3r.py +++ b/sublist3r.py @@ -1,8 +1,8 @@ -#!/usr/bin/env python +#!/usr/bin/env python4 # coding: utf-8 # Sublist3r v1.0 # By Ahmed Aboul-Ela - twitter.com/aboul3la - +# And fmjal - github.com/fmjal # modules in standard library import re import sys @@ -106,6 +106,7 @@ def parse_args(): parser.add_argument('-e', '--engines', help='Specify a comma-separated list of search engines') parser.add_argument('-o', '--output', help='Save the results to text file') parser.add_argument('-n', '--no-color', help='Output without color', default=False, action='store_true') + parser.add_argument("-s","--silent",default=False,action='store_true',help='Run without showing a banner or status updates') return parser.parse_args() @@ -1078,12 +1079,14 @@ def interactive(): enable_bruteforce = args.bruteforce verbose = args.verbose engines = args.engines + silent=args.silent if verbose or verbose is None: verbose = True if args.no_color: no_color() - banner() - res = main(domain, threads, savefile, ports, silent=False, verbose=verbose, enable_bruteforce=enable_bruteforce, engines=engines) + if not silent: + banner() + res = main(domain, threads, savefile, ports, silent=args.silent, verbose=verbose, enable_bruteforce=enable_bruteforce, engines=engines) if __name__ == "__main__": interactive() From 6bd612badc08940870a6a449abed89853d207603 Mon Sep 17 00:00:00 2001 From: user Date: Thu, 1 Aug 2024 17:22:03 +0200 Subject: [PATCH 06/10] Fix silent flag --- setup.py | 2 +- sublist3r.py | 5 ++--- 2 files changed, 3 insertions(+), 4 deletions(-) diff --git a/setup.py b/setup.py index 36b86a2..547a026 100644 --- a/setup.py +++ b/setup.py @@ -2,7 +2,7 @@ setup( name='Sublist3r', - version='2.0', + version='2.1', python_requires='>=2.7', install_requires=['dnspython', 'requests', 'argparse; python_version==\'2.7\''], packages=find_packages()+['.'], diff --git a/sublist3r.py b/sublist3r.py index f85df4a..0be8ed1 100644 --- a/sublist3r.py +++ b/sublist3r.py @@ -1064,9 +1064,8 @@ def main(domain, threads, savefile, ports, silent, verbose, enable_bruteforce, e pscan = portscan(subdomains, ports) pscan.run() - elif not silent: - for subdomain in subdomains: - print(G + subdomain + W) + for subdomain in subdomains: + print(G + subdomain + W) return subdomains From c3427fb22f6871883a1cc450e00ac012bd495da4 Mon Sep 17 00:00:00 2001 From: user Date: Thu, 1 Aug 2024 17:22:56 +0200 Subject: [PATCH 07/10] Fix silent flag --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index 547a026..d43490d 100644 --- a/setup.py +++ b/setup.py @@ -7,7 +7,7 @@ install_requires=['dnspython', 'requests', 'argparse; python_version==\'2.7\''], packages=find_packages()+['.'], include_package_data=True, - url='https://github.com/aboul3la/Sublist3r', + url='https://github.com/fmjal/Sublist3r', license='GPL-2.0', description='Subdomains enumeration tool for penetration testers', classifiers=[ From 6beb4932d7f61bfb09f6a8a40e976a5bfdc4cb6d Mon Sep 17 00:00:00 2001 From: user Date: Thu, 1 Aug 2024 17:23:49 +0200 Subject: [PATCH 08/10] Fix silent flag --- requirements.txt | 2 +- setup.py | 6 ++++-- 2 files changed, 5 insertions(+), 3 deletions(-) diff --git a/requirements.txt b/requirements.txt index 256072c..96547b8 100644 --- a/requirements.txt +++ b/requirements.txt @@ -2,4 +2,4 @@ argparse dnspython requests bs4 - +requests[socks] diff --git a/setup.py b/setup.py index d43490d..d6141ba 100644 --- a/setup.py +++ b/setup.py @@ -3,8 +3,8 @@ setup( name='Sublist3r', version='2.1', - python_requires='>=2.7', - install_requires=['dnspython', 'requests', 'argparse; python_version==\'2.7\''], + python_requires='>=3.10', + install_requires=['dnspython', 'requests', 'argparse; python_version==\'3.10\',requests[socks]'], packages=find_packages()+['.'], include_package_data=True, url='https://github.com/fmjal/Sublist3r', @@ -25,6 +25,8 @@ 'Programming Language :: Python :: 3.4', 'Programming Language :: Python :: 3.5', 'Programming Language :: Python :: 3.6', + 'Programming Language :: Python :: 3.10', + 'Programming Language :: Python :: 3.11', 'Topic :: Security', ], keywords='subdomain dns detection', From 71036875077e6521512f0258e6ac027e84ed121c Mon Sep 17 00:00:00 2001 From: Your Name Date: Fri, 2 Aug 2024 01:58:41 +0200 Subject: [PATCH 09/10] . --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index d6141ba..549270f 100644 --- a/setup.py +++ b/setup.py @@ -4,7 +4,7 @@ name='Sublist3r', version='2.1', python_requires='>=3.10', - install_requires=['dnspython', 'requests', 'argparse; python_version==\'3.10\',requests[socks]'], + install_requires=['dnspython', 'requests', 'argparse; python_version==\'3.10\''], packages=find_packages()+['.'], include_package_data=True, url='https://github.com/fmjal/Sublist3r', From b7b53d8be9a4592455c976802fd12dac0c233269 Mon Sep 17 00:00:00 2001 From: Your Name Date: Fri, 2 Aug 2024 02:00:12 +0200 Subject: [PATCH 10/10] . --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index 549270f..66746eb 100644 --- a/setup.py +++ b/setup.py @@ -4,7 +4,7 @@ name='Sublist3r', version='2.1', python_requires='>=3.10', - install_requires=['dnspython', 'requests', 'argparse; python_version==\'3.10\''], + install_requires=['bs4','dnspython', 'requests', 'argparse; python_version==\'3.10\''], packages=find_packages()+['.'], include_package_data=True, url='https://github.com/fmjal/Sublist3r',