-
Notifications
You must be signed in to change notification settings - Fork 2
/
find_tlds.py
116 lines (102 loc) · 3.88 KB
/
find_tlds.py
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
#!/usr/bin/python3
import sys
import socket
import requests
import favicon
import hashlib
import ssl
import re
import urllib3
from requests.exceptions import ConnectionError, HTTPError
from pebble import ProcessPool as ThreadPool
urllib3.disable_warnings(urllib3.exceptions.InsecureRequestWarning)
class TldScanner():
max_workers = 5
max_tasks = 15
timeout = 3
known_certs = list()
def __init__(self, domain):
try:
try:
self.tld_list = open('TLD_COMPLETE_LIST.txt', 'r').read().split('\n')
except OSError as e:
print('TLD_COMPLETE_LIST.txt is missing!\n')
print(repr(e))
exit()
self.domain = domain
self.known_ip = socket.gethostbyname(domain)
self.known_domain = domain.split('.')[0]
self.known_fav = self.check_favicon(domain)
self.known_certs.append(self.check_cert(domain))
except Exception as e:
print("Fatal error: " + repr(e))
raise
def check_cert(self, dom):
for vhost in ['', 'www.']:
try:
cert = ssl.get_server_certificate((vhost+dom, 443))
return hashlib.sha1(cert.encode('utf-8')).hexdigest()
except (OSError, socket.error):
pass
def check_favicon(self, dom):
for proto in ['http', 'https']:
try:
fav = favicon.get(proto+'://'+dom+'/', stream=True, verify=False)
r = requests.get(fav[0].url, verify=False)
return hashlib.sha1(r.text.encode('utf-8')).hexdigest()
except (ConnectionError, IndexError, HTTPError):
pass
def check_links(self, new_domain):
for vhost in ['', 'www.']:
for proto in ['http', 'https']:
try:
r = requests.get(proto+'://'+vhost+new_domain+'/', verify=False, allow_redirects=True)
if re.search(r'http.:\/\/.*'+self.domain, r.text):
return True
else:
return False
except ConnectionError:
pass
def check_tld(self, new_domain):
try:
ip = socket.gethostbyname(str(new_domain))
if str(ip) == str(self.known_ip):
try:
self.known_certs.append(self.check_cert(new_domain))
except Exception as e:
print('failed: %s' % repr(e))
pass
print('found by IPv4: %s' % new_domain)
return new_domain
elif str(self.check_cert(new_domain)) in self.known_certs:
try:
self.known_certs.append(self.check_cert(new_domain))
except Exception as e:
print('failed: %s' % repr(e))
pass
print('found by certificate: %s' % new_domain)
return new_domain
elif str(self.known_fav) == str(self.check_favicon(new_domain)):
print('found by favicon: %s' % new_domain)
return new_domain
elif self.check_links(new_domain):
print('found by link: %s' % new_domain)
return new_domain
except Exception as e:
pass
def find_tlds(self):
dom_list = [self.known_domain+'.'+tld for tld in self.tld_list]
try:
pool = ThreadPool(max_workers=self.max_workers, max_tasks=self.max_tasks)
results = pool.map(self.check_tld, dom_list, timeout=self.timeout)
pool.close()
pool.join()
print(results)
except Exception as e:
print(repr(e))
pass
if len(sys.argv) != 2:
print('\nUSAGE: %s global-domain.com\n' % str(sys.argv[0]))
else:
s = TldScanner(str(sys.argv[1]))
s.find_tlds()