Add Shodan & Hackertarget Engines

README.md

@@ -1,9 +1,15 @@
-## About Sublist3r 
+# About Sublist3r 
 
 Sublist3r is a python tool designed to enumerate subdomains of websites using OSINT. It helps penetration testers and bug hunters collect and gather subdomains for the domain they are targeting. Sublist3r enumerates subdomains using many search engines such as Google, Yahoo, Bing, Baidu and Ask. Sublist3r also enumerates subdomains using Netcraft, Virustotal, ThreatCrowd, DNSdumpster and ReverseDNS.
 
 [subbrute](https://github.com/TheRook/subbrute) was integrated with Sublist3r to increase the possibility of finding more subdomains using bruteforce with an improved wordlist. The credit goes to TheRook who is the author of subbrute.
 
+## Key Updates/Changes
+
+* Added `--silent`,`-s` flag to handle unix-like output piping
+* Added Hackertarget & Shodan Enumeraton of Domain
+* Removed Defunct Virustotal API
+
 ## Screenshots
 
 ![Sublist3r](http://www.secgeek.net/images/Sublist3r.png "Sublist3r in action")

requirements.txt

@@ -1,3 +1,5 @@
 argparse
 dnspython
 requests
+bs4
+requests[socks]

setup.py

@@ -2,12 +2,12 @@
 
 setup(
     name='Sublist3r',
-    version='1.0',
-    python_requires='>=2.7',
-    install_requires=['dnspython', 'requests', 'argparse; python_version==\'2.7\''],
+    version='2.1',
+    python_requires='>=3.10',
+    install_requires=['bs4','dnspython', 'requests', 'argparse; python_version==\'3.10\''],
     packages=find_packages()+['.'],
     include_package_data=True,
-    url='https://github.com/aboul3la/Sublist3r',
+    url='https://github.com/fmjal/Sublist3r',
     license='GPL-2.0',
     description='Subdomains enumeration tool for penetration testers',
     classifiers=[
@@ -25,6 +25,8 @@
         'Programming Language :: Python :: 3.4',
         'Programming Language :: Python :: 3.5',
         'Programming Language :: Python :: 3.6',
+        'Programming Language :: Python :: 3.10',
+        'Programming Language :: Python :: 3.11',
         'Topic :: Security',
     ],
     keywords='subdomain dns detection',

sublist3r.py

@@ -1,8 +1,8 @@
-#!/usr/bin/env python
+#!/usr/bin/env python4
 # coding: utf-8
 # Sublist3r v1.0
 # By Ahmed Aboul-Ela - twitter.com/aboul3la
-
+# And fmjal - github.com/fmjal
 # modules in standard library
 import re
 import sys
@@ -16,6 +16,7 @@
 import socket
 import json
 from collections import Counter
+from bs4 import BeautifulSoup
 
 # external modules
 from subbrute import subbrute
@@ -79,7 +80,9 @@ def banner():
                  ___) | |_| | |_) | | \__ \ |_ ___) | |
                 |____/ \__,_|_.__/|_|_|___/\__|____/|_|%s%s
 
-                # Coded By Ahmed Aboul-Ela - @aboul3la
+                # Coded By:
+                # - Ahmed Aboul-Ela - @aboul3la
+                # - fmjal - @fmjal
     """ % (R, W, Y))
 
 
@@ -103,6 +106,7 @@ def parse_args():
     parser.add_argument('-e', '--engines', help='Specify a comma-separated list of search engines')
     parser.add_argument('-o', '--output', help='Save the results to text file')
     parser.add_argument('-n', '--no-color', help='Output without color', default=False, action='store_true')
+    parser.add_argument("-s","--silent",default=False,action='store_true',help='Run without showing a banner or status updates')
     return parser.parse_args()
 
 
@@ -850,6 +854,84 @@ def extract_domains(self, resp):
         except Exception as e:
             pass
 
+class HTEnum(enumratorBaseThreaded):
+    def __init__(self, domain, subdomains=None, q=None, silent=False, verbose=True):
+        subdomains = subdomains or []
+        base_url = 'https://api.hackertarget.com/hostsearch/?q={domain}'
+        self.engine_name = "HTEnum"
+        self.q = q
+        super(HTEnum, self).__init__(base_url, self.engine_name, domain, subdomains, q=q, silent=silent, verbose=verbose)
+        return
+
+    def req(self, url):
+        try:
+            resp = self.session.get(url, headers=self.headers, timeout=self.timeout)
+        except Exception as e:
+            resp = None
+
+        return self.get_response(resp)
+
+    def get_response(self, resp):
+        if resp and resp.status_code == 200 and not resp.text.startswith("API Count"):
+            return resp.text
+        return None
+
+    def extract_domains(self, resp):
+        for line in resp.splitlines():
+            subdomain = line.split(',')[0]
+            if subdomain and subdomain not in self.subdomains:
+                self.subdomains.append(subdomain)
+
+    def enumerate(self):
+        url = self.base_url.format(domain=self.domain)
+        resp = self.req(url)
+        if not resp:
+            return self.subdomains
+
+        self.extract_domains(resp)
+        return self.subdomains
+
+class ShodanEnum(enumratorBaseThreaded):
+    def __init__(self, domain, subdomains=None, q=None, silent=False, verbose=True):
+        self.domain=domain
+        subdomains = subdomains or []
+        base_url = 'https://www.shodan.io/domain/{domain}'
+        self.engine_name = "Shodan"
+        self.q = q
+        super(ShodanEnum, self).__init__(base_url, self.engine_name, domain, subdomains, q=q, silent=silent, verbose=verbose)
+
+    def req(self, url):
+        try:
+            resp = self.session.get(url, headers=self.headers, timeout=self.timeout)
+        except Exception as e:
+            resp = None
+
+        return self.get_response(resp)
+
+    def get_response(self, resp):
+        if resp and resp.status_code == 200:
+            return resp.text
+        return None
+
+    def extract_domains(self, resp):
+        soup = BeautifulSoup(resp, 'html.parser')
+        subdomains_list = soup.find('ul', id='subdomains')
+        if subdomains_list:
+            for li in subdomains_list.find_all('li'):
+                subdomain = li.get_text().strip()
+                if subdomain and subdomain not in self.subdomains:
+                    if subdomain != "*" and subdomain != "_dmarc":
+                        self.subdomains.append(f'{subdomain}.{self.domain}')
+
+    def enumerate(self):
+        url = self.base_url.format(domain=self.domain)
+        resp = self.req(url)
+        if not resp:
+            return self.subdomains
+
+        self.extract_domains(resp)
+        return self.subdomains
+
 
 class portscan():
     def __init__(self, subdomains, ports):
@@ -922,16 +1004,18 @@ def main(domain, threads, savefile, ports, silent, verbose, enable_bruteforce, e
                          'virustotal': Virustotal,
                          'threatcrowd': ThreatCrowd,
                          'ssl': CrtSearch,
-                         'passivedns': PassiveDNS
+                         'passivedns': PassiveDNS,
+                         "HTEnum":HTEnum,
+                         "Shodan":ShodanEnum
                          }
 
     chosenEnums = []
 
     if engines is None:
         chosenEnums = [
             BaiduEnum, YahooEnum, GoogleEnum, BingEnum, AskEnum,
-            NetcraftEnum, DNSdumpster, Virustotal, ThreatCrowd,
-            CrtSearch, PassiveDNS
+            NetcraftEnum, DNSdumpster, ThreatCrowd,
+            CrtSearch, PassiveDNS,HTEnum,ShodanEnum
         ]
     else:
         engines = engines.split(',')
@@ -980,9 +1064,8 @@ def main(domain, threads, savefile, ports, silent, verbose, enable_bruteforce, e
             pscan = portscan(subdomains, ports)
             pscan.run()
 
-        elif not silent:
-            for subdomain in subdomains:
-                print(G + subdomain + W)
+        for subdomain in subdomains:
+            print(G + subdomain + W)
     return subdomains
 
 
@@ -995,12 +1078,14 @@ def interactive():
     enable_bruteforce = args.bruteforce
     verbose = args.verbose
     engines = args.engines
+    silent=args.silent
     if verbose or verbose is None:
         verbose = True
     if args.no_color:
         no_color()
-    banner()
-    res = main(domain, threads, savefile, ports, silent=False, verbose=verbose, enable_bruteforce=enable_bruteforce, engines=engines)
+    if not silent:
+        banner()
+    res = main(domain, threads, savefile, ports, silent=args.silent, verbose=verbose, enable_bruteforce=enable_bruteforce, engines=engines)
 
 if __name__ == "__main__":
     interactive()