diff --git a/CHANGELOG.md b/CHANGELOG.md index 1e3220e..2e25cdd 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,12 @@ # Whoiser change log +#### 1.13.3 - 1 December 2022 + +- Added - Support for third level domains [#80](https://github.com/LayeredStudio/whoiser/pull/80) +- Added - Additional support for TLDs not in the IANA database [#80](https://github.com/LayeredStudio/whoiser/pull/80) +- Fixed - Follow RIPE referrals [#80](https://github.com/LayeredStudio/whoiser/pull/80) +- Fixed - Parse .gg, .je, and .as whois data correctly [#80](https://github.com/LayeredStudio/whoiser/pull/80) + #### 1.13.2 - 28 November 2022 - Updated - Include more WHOIS servers in lib, speeds-up domain WHOIS queries diff --git a/package.json b/package.json index bdffbbf..fdbd80e 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "whoiser", - "version": "1.13.2", + "version": "1.13.3", "description": "Whois info for TLDs, domains and IPs", "types": "./index.d.ts", "typings": "./index.d.ts", diff --git a/src/parsers.js b/src/parsers.js index 0c26e34..4b57dc7 100644 --- a/src/parsers.js +++ b/src/parsers.js @@ -265,13 +265,26 @@ const parseDomainWhois = (domain, whois) => { .split('\n') .map((line) => line.replace('\t', ' ')) - // Parse WHOIS info for specific TLDs - if (domain.endsWith('.uk') || domain.endsWith('.be') || domain.endsWith('.nl') || domain.endsWith('.eu') || domain.endsWith('.ly') || domain.endsWith('.mx')) { + if ( + domain.endsWith('.uk') || + domain.endsWith('.be') || + domain.endsWith('.nl') || + domain.endsWith('.eu') || + domain.endsWith('.ly') || + domain.endsWith('.mx') || + domain.endsWith('.gg') || + domain.endsWith('.je') || + domain.endsWith('.as') + ) { lines = handleMultiLines(lines) } + if (domain.endsWith('.gg') || domain.endsWith('.je') || domain.endsWith('.as')) { + lines = handleMissingColons(lines) + } + if (domain.endsWith('.ua')) { lines = handleDotUa(lines) colon = ':' @@ -307,14 +320,12 @@ const parseDomainWhois = (domain, whois) => { if (data[label] && Array.isArray(data[label])) { data[label].push(value) } else if (!ignoreLabels.includes(label.toLowerCase()) && !ignoreTexts.some((text) => label.toLowerCase().includes(text))) { - // WHOIS field already exists, if so append data if (data[label] && data[label] !== value) { data[label] = `${data[label]} ${value}`.trim() } else { data[label] = value } - } else { text.push(line) } @@ -411,13 +422,13 @@ const handleJpLines = (lines) => { line = line.replace(/^[a-z]. \[/, '[') } - if (line.startsWith("[ ")) { + if (line.startsWith('[ ')) { // skip - } else if (line.startsWith("[")) { + } else if (line.startsWith('[')) { ret.push(line) - } else if (line.startsWith(" ")) { + } else if (line.startsWith(' ')) { const prev = ret.pop() - ret.push(prev + "\n" + line.trim()) + ret.push(prev + '\n' + line.trim()) } else { // skip } @@ -425,5 +436,17 @@ const handleJpLines = (lines) => { return ret.map((line) => line.replace(/\[(.*?)\]/g, '$1:')) } +// Handle formats like this: +// Registrar Gandi SAS +const handleMissingColons = (lines) => { + lines.forEach((line, index) => { + if (line.startsWith('Registrar ')) { + lines[index] = line.replace('Registrar ', 'Registrar: ') + } + }) + + return lines +} + module.exports.parseSimpleWhois = parseSimpleWhois module.exports.parseDomainWhois = parseDomainWhois diff --git a/src/utils.js b/src/utils.js index a47df2c..e00fca4 100644 --- a/src/utils.js +++ b/src/utils.js @@ -2,12 +2,12 @@ const punycode = require('punycode') const https = require('https') const splitStringBy = (string, by) => [string.slice(0, by), string.slice(by + 1)] -const requestGetBody = url => { +const requestGetBody = (url) => { return new Promise((resolve, reject) => { https - .get(url, resp => { + .get(url, (resp) => { let data = '' - resp.on('data', chunk => (data += chunk)) + resp.on('data', (chunk) => (data += chunk)) resp.on('end', () => resolve(data)) resp.on('error', reject) }) @@ -15,7 +15,7 @@ const requestGetBody = url => { }) } -const isTld = tld => { +const isTld = (tld) => { if (tld.startsWith('.')) { tld = tld.substring(1) } @@ -23,15 +23,12 @@ const isTld = tld => { return /^([a-z]{2,64}|xn[a-z0-9-]{5,})$/i.test(punycode.toASCII(tld)) } -const isDomain = domain => { +const isDomain = (domain) => { if (domain.endsWith('.')) { domain = domain.substring(0, domain.length - 1) } - const labels = punycode - .toASCII(domain) - .split('.') - .reverse() + const labels = punycode.toASCII(domain).split('.').reverse() const labelTest = /^([a-z0-9-]{1,64}|xn[a-z0-9-]{5,})$/i return ( diff --git a/src/whoiser.js b/src/whoiser.js index a87bc91..de0a097 100644 --- a/src/whoiser.js +++ b/src/whoiser.js @@ -1,5 +1,6 @@ const net = require('net') const url = require('url') +const dns = require('dns/promises') const punycode = require('punycode') const { parseSimpleWhois, parseDomainWhois } = require('./parsers.js') const { splitStringBy, requestGetBody, isTld, isDomain } = require('./utils.js') @@ -72,7 +73,32 @@ const allTlds = async () => { return tlds.split('\n').filter((tld) => Boolean(tld) && !tld.startsWith('#')) } -const whoisTld = async (query, { timeout = 15000, raw = false } = {}) => { +const whoisTldAlternate = async (query) => { + const [whoisCname, whoisSrv] = await Promise.allSettled([ + // Check sources for whois server + dns.resolveCname(`${query}.whois-servers.net`), // Queries public database for whois server + dns.resolveSrv(`_nicname._tcp.${query}`), // Queries for whois server published by registry + ]) + + return whoisSrv?.value?.[0]?.name ?? whoisCname?.value?.[0] // Get whois server from results +} + +const whoisTld = async (query, { timeout = 15000, raw = false, domainThirdLevel = false, domainName = '', domainTld = '' } = {}) => { + // Check for 3rd level domain + if (domainThirdLevel) { + let [_, secondTld] = domainName && splitStringBy(domainName, domainName.lastIndexOf('.')) // Parse 3rd level domain + const finalTld = secondTld ? `${secondTld}.${domainTld}` : query + + const whois = await whoisTldAlternate(finalTld) // Query alternate sources + if (whois) + return { + refer: whois, + domain: domainName, + finalTld, + whois, + } // Return alternate whois data + } + const result = await whoisQuery({ host: 'whois.iana.org', query, timeout }) const data = parseSimpleWhois(result) @@ -80,7 +106,15 @@ const whoisTld = async (query, { timeout = 15000, raw = false } = {}) => { data.__raw = result } - if (!data.domain || !data.domain.length) { + if (!data.domain || !data.domain.length || !data.whois) { + const whois = await whoisTldAlternate(domainTld) // Query alternate sources + if (whois) + return { + refer: whois, + domain: domainName, + whois, + } // Return alternate whois data + throw new Error(`TLD "${query}" not found`) } @@ -89,24 +123,25 @@ const whoisTld = async (query, { timeout = 15000, raw = false } = {}) => { const whoisDomain = async (domain, { host = null, timeout = 15000, follow = 2, raw = false } = {}) => { domain = punycode.toASCII(domain) + const domainThirdLevel = domain.lastIndexOf('.') !== domain.indexOf('.') const [domainName, domainTld] = splitStringBy(domain.toLowerCase(), domain.lastIndexOf('.')) let results = {} // find WHOIS server in cache - if (!host && cacheTldWhoisServer[domainTld]) { + if (!host && !domainThirdLevel && cacheTldWhoisServer[domainTld]) { host = cacheTldWhoisServer[domainTld] } // find WHOIS server for TLD if (!host) { - const tld = await whoisTld(domain, { timeout }) + const tld = await whoisTld(domain, { timeout, domainThirdLevel, domainName, domainTld }) if (!tld.whois) { throw new Error(`TLD for "${domain}" not supported`) } host = tld.whois - cacheTldWhoisServer[domainTld] = tld.whois + cacheTldWhoisServer[tld.finalTld || domainTld] = tld.whois } // query WHOIS servers for data @@ -172,7 +207,7 @@ const whoisDomain = async (domain, { host = null, timeout = 15000, follow = 2, r return results } -const whoisIpOrAsn = async (query, { host = null, timeout = 15000, raw = false } = {}) => { +const whoisIpOrAsn = async (query, { host = null, timeout = 15000, follow = 2, raw = false } = {}) => { const type = net.isIP(query) ? 'ip' : 'asn' query = String(query) @@ -190,18 +225,27 @@ const whoisIpOrAsn = async (query, { host = null, timeout = 15000, raw = false } throw new Error(`No WHOIS server for "${query}"`) } - // hardcoded custom queries.. - if (host === 'whois.arin.net' && type === 'ip') { - query = `+ n ${query}` - } else if (host === 'whois.arin.net' && type === 'asn') { - query = `+ a ${query}` - } + let data + + while (host && follow) { + let modifiedQuery = query + + // hardcoded custom queries.. + if (host === 'whois.arin.net' && type === 'ip') { + modifiedQuery = `+ n ${query}` + } else if (host === 'whois.arin.net' && type === 'asn') { + modifiedQuery = `+ a ${query}` + } - const rawResult = await whoisQuery({ host, query, timeout }) - let data = parseSimpleWhois(rawResult) + const rawResult = await whoisQuery({ host, query: modifiedQuery, timeout }) + data = parseSimpleWhois(rawResult) - if (raw) { - data.__raw = rawResult + if (raw) { + data.__raw = rawResult + } + + follow-- + host = data?.ReferralServer?.split('//')?.[1] } return data diff --git a/test/domains.js b/test/domains.js index 2da9a05..5c808c4 100644 --- a/test/domains.js +++ b/test/domains.js @@ -12,6 +12,14 @@ describe('#whoiser.domain()', function() { assert.equal(firstWhois['Registry Domain ID'], '27CAA9F68-GOOGLE', 'Registry Domain ID doesn\'t match') }); + it('returns WHOIS for "google.co.uk"', async function() { + const whois = await whoiser.domain('google.co.uk') + const firstWhois = whoiser.firstResult(whois) + + assert.equal(firstWhois['Domain Name'], 'google.co.uk', 'Domain name doesn\'t match') + assert.equal(firstWhois['Created Date'], '14-Feb-1999', 'Created Date doesn\'t match') + }); + it('returns WHOIS for "cloudflare.com" from "whois.cloudflare.com" server (host option)', async function() { let whois = await whoiser.domain('cloudflare.com', {host: 'whois.cloudflare.com'}) assert.equal(Object.values(whois).length, 1, 'Has less or more than 1 WHOIS result')