Merge pull request #80 from popeeyy/master

Closes #50
Closes #72

CHANGELOG.md

@@ -1,5 +1,12 @@
 # Whoiser change log
 
+#### 1.13.3 - 1 December 2022
+
+- Added - Support for third level domains [#80](https://github.com/LayeredStudio/whoiser/pull/80)
+- Added - Additional support for TLDs not in the IANA database [#80](https://github.com/LayeredStudio/whoiser/pull/80)
+- Fixed - Follow RIPE referrals [#80](https://github.com/LayeredStudio/whoiser/pull/80)
+- Fixed - Parse .gg, .je, and .as whois data correctly [#80](https://github.com/LayeredStudio/whoiser/pull/80)
+
 #### 1.13.2 - 28 November 2022
 
 - Updated - Include more WHOIS servers in lib, speeds-up domain WHOIS queries

package.json

@@ -1,6 +1,6 @@
 {
   "name": "whoiser",
-  "version": "1.13.2",
+  "version": "1.13.3",
   "description": "Whois info for TLDs, domains and IPs",
   "types": "./index.d.ts",
   "typings": "./index.d.ts",

src/parsers.js

@@ -265,13 +265,26 @@ const parseDomainWhois = (domain, whois) => {
 		.split('\n')
 		.map((line) => line.replace('\t', '  '))
 
-
 	// Parse WHOIS info for specific TLDs
 
-	if (domain.endsWith('.uk') || domain.endsWith('.be') || domain.endsWith('.nl') || domain.endsWith('.eu') || domain.endsWith('.ly') || domain.endsWith('.mx')) {
+	if (
+		domain.endsWith('.uk') ||
+		domain.endsWith('.be') ||
+		domain.endsWith('.nl') ||
+		domain.endsWith('.eu') ||
+		domain.endsWith('.ly') ||
+		domain.endsWith('.mx') ||
+		domain.endsWith('.gg') ||
+		domain.endsWith('.je') ||
+		domain.endsWith('.as')
+	) {
 		lines = handleMultiLines(lines)
 	}
 
+	if (domain.endsWith('.gg') || domain.endsWith('.je') || domain.endsWith('.as')) {
+		lines = handleMissingColons(lines)
+	}
+
 	if (domain.endsWith('.ua')) {
 		lines = handleDotUa(lines)
 		colon = ':'
@@ -307,14 +320,12 @@ const parseDomainWhois = (domain, whois) => {
 			if (data[label] && Array.isArray(data[label])) {
 				data[label].push(value)
 			} else if (!ignoreLabels.includes(label.toLowerCase()) && !ignoreTexts.some((text) => label.toLowerCase().includes(text))) {
-
 				// WHOIS field already exists, if so append data
 				if (data[label] && data[label] !== value) {
 					data[label] = `${data[label]} ${value}`.trim()
 				} else {
 					data[label] = value
 				}
-
 			} else {
 				text.push(line)
 			}
@@ -411,19 +422,31 @@ const handleJpLines = (lines) => {
 			line = line.replace(/^[a-z]. \[/, '[')
 		}
 
-		if (line.startsWith("[ ")) {
+		if (line.startsWith('[ ')) {
 			// skip
-		} else if (line.startsWith("[")) {
+		} else if (line.startsWith('[')) {
 			ret.push(line)
-		} else if (line.startsWith(" ")) {
+		} else if (line.startsWith(' ')) {
 			const prev = ret.pop()
-			ret.push(prev + "\n" + line.trim())
+			ret.push(prev + '\n' + line.trim())
 		} else {
 			// skip
 		}
 	}
 	return ret.map((line) => line.replace(/\[(.*?)\]/g, '$1:'))
 }
 
+// Handle formats like this:
+// Registrar Gandi SAS
+const handleMissingColons = (lines) => {
+	lines.forEach((line, index) => {
+		if (line.startsWith('Registrar ')) {
+			lines[index] = line.replace('Registrar ', 'Registrar: ')
+		}
+	})
+
+	return lines
+}
+
 module.exports.parseSimpleWhois = parseSimpleWhois
 module.exports.parseDomainWhois = parseDomainWhois

src/utils.js

@@ -2,36 +2,33 @@ const punycode = require('punycode')
 const https = require('https')
 const splitStringBy = (string, by) => [string.slice(0, by), string.slice(by + 1)]
 
-const requestGetBody = url => {
+const requestGetBody = (url) => {
 	return new Promise((resolve, reject) => {
 		https
-			.get(url, resp => {
+			.get(url, (resp) => {
 				let data = ''
-				resp.on('data', chunk => (data += chunk))
+				resp.on('data', (chunk) => (data += chunk))
 				resp.on('end', () => resolve(data))
 				resp.on('error', reject)
 			})
 			.on('error', reject)
 	})
 }
 
-const isTld = tld => {
+const isTld = (tld) => {
 	if (tld.startsWith('.')) {
 		tld = tld.substring(1)
 	}
 
 	return /^([a-z]{2,64}|xn[a-z0-9-]{5,})$/i.test(punycode.toASCII(tld))
 }
 
-const isDomain = domain => {
+const isDomain = (domain) => {
 	if (domain.endsWith('.')) {
 		domain = domain.substring(0, domain.length - 1)
 	}
 
-	const labels = punycode
-		.toASCII(domain)
-		.split('.')
-		.reverse()
+	const labels = punycode.toASCII(domain).split('.').reverse()
 	const labelTest = /^([a-z0-9-]{1,64}|xn[a-z0-9-]{5,})$/i
 
 	return (

src/whoiser.js

@@ -1,5 +1,6 @@
 const net = require('net')
 const url = require('url')
+const dns = require('dns/promises')
 const punycode = require('punycode')
 const { parseSimpleWhois, parseDomainWhois } = require('./parsers.js')
 const { splitStringBy, requestGetBody, isTld, isDomain } = require('./utils.js')
@@ -72,15 +73,48 @@ const allTlds = async () => {
 	return tlds.split('\n').filter((tld) => Boolean(tld) && !tld.startsWith('#'))
 }
 
-const whoisTld = async (query, { timeout = 15000, raw = false } = {}) => {
+const whoisTldAlternate = async (query) => {
+	const [whoisCname, whoisSrv] = await Promise.allSettled([
+		// Check sources for whois server
+		dns.resolveCname(`${query}.whois-servers.net`), // Queries public database for whois server
+		dns.resolveSrv(`_nicname._tcp.${query}`), // Queries for whois server published by registry
+	])
+
+	return whoisSrv?.value?.[0]?.name ?? whoisCname?.value?.[0] // Get whois server from results
+}
+
+const whoisTld = async (query, { timeout = 15000, raw = false, domainThirdLevel = false, domainName = '', domainTld = '' } = {}) => {
+	// Check for 3rd level domain
+	if (domainThirdLevel) {
+		let [_, secondTld] = domainName && splitStringBy(domainName, domainName.lastIndexOf('.')) // Parse 3rd level domain
+		const finalTld = secondTld ? `${secondTld}.${domainTld}` : query
+
+		const whois = await whoisTldAlternate(finalTld) // Query alternate sources
+		if (whois)
+			return {
+				refer: whois,
+				domain: domainName,
+				finalTld,
+				whois,
+			} // Return alternate whois data
+	}
+
 	const result = await whoisQuery({ host: 'whois.iana.org', query, timeout })
 	const data = parseSimpleWhois(result)
 
 	if (raw) {
 		data.__raw = result
 	}
 
-	if (!data.domain || !data.domain.length) {
+	if (!data.domain || !data.domain.length || !data.whois) {
+		const whois = await whoisTldAlternate(domainTld) // Query alternate sources
+		if (whois)
+			return {
+				refer: whois,
+				domain: domainName,
+				whois,
+			} // Return alternate whois data
+
 		throw new Error(`TLD "${query}" not found`)
 	}
 
@@ -89,24 +123,25 @@ const whoisTld = async (query, { timeout = 15000, raw = false } = {}) => {
 
 const whoisDomain = async (domain, { host = null, timeout = 15000, follow = 2, raw = false } = {}) => {
 	domain = punycode.toASCII(domain)
+	const domainThirdLevel = domain.lastIndexOf('.') !== domain.indexOf('.')
 	const [domainName, domainTld] = splitStringBy(domain.toLowerCase(), domain.lastIndexOf('.'))
 	let results = {}
 
 	// find WHOIS server in cache
-	if (!host && cacheTldWhoisServer[domainTld]) {
+	if (!host && !domainThirdLevel && cacheTldWhoisServer[domainTld]) {
 		host = cacheTldWhoisServer[domainTld]
 	}
 
 	// find WHOIS server for TLD
 	if (!host) {
-		const tld = await whoisTld(domain, { timeout })
+		const tld = await whoisTld(domain, { timeout, domainThirdLevel, domainName, domainTld })
 
 		if (!tld.whois) {
 			throw new Error(`TLD for "${domain}" not supported`)
 		}
 
 		host = tld.whois
-		cacheTldWhoisServer[domainTld] = tld.whois
+		cacheTldWhoisServer[tld.finalTld || domainTld] = tld.whois
 	}
 
 	// query WHOIS servers for data
@@ -172,7 +207,7 @@ const whoisDomain = async (domain, { host = null, timeout = 15000, follow = 2, r
 	return results
 }
 
-const whoisIpOrAsn = async (query, { host = null, timeout = 15000, raw = false } = {}) => {
+const whoisIpOrAsn = async (query, { host = null, timeout = 15000, follow = 2, raw = false } = {}) => {
 	const type = net.isIP(query) ? 'ip' : 'asn'
 	query = String(query)
 
@@ -190,18 +225,27 @@ const whoisIpOrAsn = async (query, { host = null, timeout = 15000, raw = false }
 		throw new Error(`No WHOIS server for "${query}"`)
 	}
 
-	// hardcoded custom queries..
-	if (host === 'whois.arin.net' && type === 'ip') {
-		query = `+ n ${query}`
-	} else if (host === 'whois.arin.net' && type === 'asn') {
-		query = `+ a ${query}`
-	}
+	let data
+
+	while (host && follow) {
+		let modifiedQuery = query
+
+		// hardcoded custom queries..
+		if (host === 'whois.arin.net' && type === 'ip') {
+			modifiedQuery = `+ n ${query}`
+		} else if (host === 'whois.arin.net' && type === 'asn') {
+			modifiedQuery = `+ a ${query}`
+		}
 
-	const rawResult = await whoisQuery({ host, query, timeout })
-	let data = parseSimpleWhois(rawResult)
+		const rawResult = await whoisQuery({ host, query: modifiedQuery, timeout })
+		data = parseSimpleWhois(rawResult)
 
-	if (raw) {
-		data.__raw = rawResult
+		if (raw) {
+			data.__raw = rawResult
+		}
+
+		follow--
+		host = data?.ReferralServer?.split('//')?.[1]
 	}
 
 	return data

test/domains.js

@@ -12,6 +12,14 @@ describe('#whoiser.domain()', function() {
 			assert.equal(firstWhois['Registry Domain ID'], '27CAA9F68-GOOGLE', 'Registry Domain ID doesn\'t match')
 		});
 
+		it('returns WHOIS for "google.co.uk"', async function() {
+			const whois = await whoiser.domain('google.co.uk')
+			const firstWhois = whoiser.firstResult(whois)
+
+			assert.equal(firstWhois['Domain Name'], 'google.co.uk', 'Domain name doesn\'t match')
+			assert.equal(firstWhois['Created Date'], '14-Feb-1999', 'Created Date doesn\'t match')
+		});
+
 		it('returns WHOIS for "cloudflare.com" from "whois.cloudflare.com" server (host option)', async function() {
 			let whois = await whoiser.domain('cloudflare.com', {host: 'whois.cloudflare.com'})
 			assert.equal(Object.values(whois).length, 1, 'Has less or more than 1 WHOIS result')