adding auto-root domains extraction (#333)

* adding auto-root domains extraction

* adding test

* dep update

---------

Co-authored-by: Sandeep Singh <[email protected]>
Co-authored-by: sandeep <[email protected]>

.github/workflows/build-test.yml

@@ -36,6 +36,10 @@ jobs:
           sudo make install
         
       # Tests
-      - name: Basic Usage
+      - name: Bruteforce Mode Test
         run: go run . -v -d scanme.sh -r ../../tests/resolvers.txt -w ../../tests/wordlist.txt -mode bruteforce
         working-directory: cmd/shuffledns/
+
+      - name: Resolve Mode Test
+        run: go run . -l ../../tests/subdomains.txt -v -d wrongdomain.sh -r ../../tests/resolvers.txt -mode resolve -auto-domain
+        working-directory: cmd/shuffledns/

README.md

@@ -60,6 +60,7 @@ Flags:
 Flags:
 INPUT:
    -d, -domain string[]           Domain to find or resolve subdomains for
+   -ad, -auto-domain              Automatically extract root domains
    -l, -list string               File containing list of subdomains to resolve
    -w, -wordlist string           File containing words to bruteforce for domain
    -r, -resolver string           File containing list of resolvers for enumeration

go.mod

@@ -12,6 +12,7 @@ require (
 	github.com/rs/xid v1.5.0
 	github.com/stretchr/testify v1.9.0
 	github.com/syndtr/goleveldb v1.0.0
+	github.com/weppos/publicsuffix-go v0.30.2
 )
 
 require (
@@ -82,7 +83,6 @@ require (
 	github.com/tklauser/go-sysconf v0.3.12 // indirect
 	github.com/tklauser/numcpus v0.6.1 // indirect
 	github.com/ulikunitz/xz v0.5.11 // indirect
-	github.com/weppos/publicsuffix-go v0.30.1 // indirect
 	github.com/xi2/xz v0.0.0-20171230120015-48954b6210f8 // indirect
 	github.com/yl2chen/cidranger v1.0.2 // indirect
 	github.com/yuin/goldmark v1.5.4 // indirect

go.sum

@@ -195,8 +195,8 @@ github.com/ulikunitz/xz v0.5.8/go.mod h1:nbz6k7qbPmH4IRqmfOplQw/tblSgqTqBwxkY0oW
 github.com/ulikunitz/xz v0.5.9/go.mod h1:nbz6k7qbPmH4IRqmfOplQw/tblSgqTqBwxkY0oWt/14=
 github.com/ulikunitz/xz v0.5.11 h1:kpFauv27b6ynzBNT/Xy+1k+fK4WswhN/6PN5WhFAGw8=
 github.com/ulikunitz/xz v0.5.11/go.mod h1:nbz6k7qbPmH4IRqmfOplQw/tblSgqTqBwxkY0oWt/14=
-github.com/weppos/publicsuffix-go v0.30.1 h1:8q+QwBS1MY56Zjfk/50ycu33NN8aa1iCCEQwo/71Oos=
-github.com/weppos/publicsuffix-go v0.30.1/go.mod h1:s41lQh6dIsDWIC1OWh7ChWJXLH0zkJ9KHZVqA7vHyuQ=
+github.com/weppos/publicsuffix-go v0.30.2 h1:Np18yzfMR90jNampWFs7iSh2sw/qCZkhL41/ffyihCU=
+github.com/weppos/publicsuffix-go v0.30.2/go.mod h1:/hGscit36Yt+wammfBBwdMdxBT8btsTt6KvwO9OvMyM=
 github.com/xi2/xz v0.0.0-20171230120015-48954b6210f8 h1:nIPpBwaJSVYIxUFsDv3M8ofmx9yWTog9BfvIu0q41lo=
 github.com/xi2/xz v0.0.0-20171230120015-48954b6210f8/go.mod h1:HUYIGzjTL3rfEspMxjDjgmT5uz5wzYJKVo23qUhYTos=
 github.com/yl2chen/cidranger v1.0.2 h1:lbOWZVCG1tCRX4u24kuM1Tb4nHqWkDxwLdoS+SevawU=
@@ -220,7 +220,7 @@ golang.org/x/crypto v0.0.0-20210220033148-5ea612d1eb83/go.mod h1:jdWPYTVW3xRLrWP
 golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc=
 golang.org/x/crypto v0.0.0-20211209193657-4570a0811e8b/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4=
 golang.org/x/crypto v0.7.0/go.mod h1:pYwdfH91IfpZVANVyUOhSIPZaFoJGxTFbZhFTx+dXZU=
-golang.org/x/crypto v0.11.0/go.mod h1:xgJhtzW8F9jGdVFWZESrid1U1bjeNy4zgy5cRr/CIio=
+golang.org/x/crypto v0.14.0/go.mod h1:MVFd36DqK4CsrnJYDkBA3VC4m2GkXAM0PvzMCn4JQf4=
 golang.org/x/crypto v0.21.0 h1:X31++rzVUdKhX5sWmSOFZxx8UW/ldWx55cbf08iNAMA=
 golang.org/x/crypto v0.21.0/go.mod h1:0BP7YvVV9gBbVKyeTG0Gyn+gZm94bibOW5BjDEYAOMs=
 golang.org/x/exp v0.0.0-20230420155640-133eef4313cb h1:rhjz/8Mbfa8xROFiH+MQphmAmgqRM0bOMnytznhWEXk=
@@ -241,7 +241,7 @@ golang.org/x/net v0.0.0-20221002022538-bcab6841153b/go.mod h1:YDH+HFinaLZZlnHAfS
 golang.org/x/net v0.6.0/go.mod h1:2Tu9+aMcznHK/AK1HMvgo6xiTLG5rD5rZLDS+rp2Bjs=
 golang.org/x/net v0.8.0/go.mod h1:QVkue5JL9kW//ek3r6jTKnTFis1tRmNAW2P1shuFdJc=
 golang.org/x/net v0.10.0/go.mod h1:0qNGK6F8kojg2nk9dLZ2mShWaEBan6FAoqfSigmmuDg=
-golang.org/x/net v0.12.0/go.mod h1:zEVYFnQC7m/vmpQFELhcD1EWkZlX69l4oqgmer6hfKA=
+golang.org/x/net v0.17.0/go.mod h1:NxSsAGuq816PNPmqtQdLE42eU2Fs7NoRIZrHJAlaCOE=
 golang.org/x/net v0.23.0 h1:7EYJ93RZ9vYSZAIb2x3lnuvqO5zneoD6IvWjuhfxjTs=
 golang.org/x/net v0.23.0/go.mod h1:JKghWKKOSdJwpW2GEx0Ja7fmaKnMsbu+MWVZTokSYmg=
 golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U=
@@ -274,6 +274,7 @@ golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.8.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.10.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.11.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
+golang.org/x/sys v0.13.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.18.0 h1:DBdB3niSjOA/O0blCZBqDefyWNYveAYMNF1Wum0DYQ4=
 golang.org/x/sys v0.18.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
 golang.org/x/term v0.0.0-20201117132131-f5c789dd3221/go.mod h1:Nr5EML6q2oocZ2LXRh80K7BxOlk5/8JxuGnuhpl+muw=
@@ -282,7 +283,7 @@ golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuX
 golang.org/x/term v0.5.0/go.mod h1:jMB1sMXY+tzblOD4FWmEbocvup2/aLOaQEp7JmGp78k=
 golang.org/x/term v0.6.0/go.mod h1:m6U89DPEgQRMq3DNkDClhWw02AUbt2daBVO4cn4Hv9U=
 golang.org/x/term v0.8.0/go.mod h1:xPskH00ivmX89bAKVGSKKtLOWNx2+17Eiy94tnKShWo=
-golang.org/x/term v0.10.0/go.mod h1:lpqdcUyK/oCiQxvxVrppt5ggO2KCZ5QblwqPnfZ6d5o=
+golang.org/x/term v0.13.0/go.mod h1:LTmsnFJwVN6bCy1rVCoS+qHT1HhALEFxKncY3WNNh4U=
 golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
 golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk=
 golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
@@ -291,7 +292,7 @@ golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ=
 golang.org/x/text v0.7.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8=
 golang.org/x/text v0.8.0/go.mod h1:e1OnstbJyHTd6l/uOt8jFFHp6TRDWZR/bV3emEE/zU8=
 golang.org/x/text v0.9.0/go.mod h1:e1OnstbJyHTd6l/uOt8jFFHp6TRDWZR/bV3emEE/zU8=
-golang.org/x/text v0.11.0/go.mod h1:TvPlkZtksWOMsz7fbANvkp4WM8x/WCo/om8BMLbz+aE=
+golang.org/x/text v0.13.0/go.mod h1:TvPlkZtksWOMsz7fbANvkp4WM8x/WCo/om8BMLbz+aE=
 golang.org/x/text v0.14.0 h1:ScX5w1eTa3QqT8oi6+ziP7dTV1S2+ALU0bI+0zXKWiQ=
 golang.org/x/text v0.14.0/go.mod h1:18ZOQIKpY8NJVqYksKHtTdi31H5itFRjB5/qKTNYzSU=
 golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
@@ -308,7 +309,6 @@ google.golang.org/appengine v1.6.7/go.mod h1:8WjMMxjGQR8xUklV/ARdw2HLXBOI7O7uCID
 google.golang.org/protobuf v1.26.0-rc.1/go.mod h1:jlhhOSvTdKEhbULTjvd4ARK9grFBp09yW+WbY/TyQbw=
 google.golang.org/protobuf v1.26.0/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc=
 google.golang.org/protobuf v1.28.0/go.mod h1:HV8QOd/L58Z+nl8r43ehVNZIU/HEI6OcFqwMG9pJV4I=
-google.golang.org/protobuf v1.28.1/go.mod h1:HV8QOd/L58Z+nl8r43ehVNZIU/HEI6OcFqwMG9pJV4I=
 google.golang.org/protobuf v1.33.0 h1:uNO2rsAINq/JlFpSdYEKIZ0uKD/R9cpdv0T+yoGwGmI=
 google.golang.org/protobuf v1.33.0/go.mod h1:c6P6GXX6sHbq/GpV6MGZEdwhWPcYBgnhAHhKbcUYpos=
 gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405 h1:yhCVgyC4o1eVCa2tZl7eS0r+SDo693bJlVdllGtEeKM=

pkg/massdns/massdns.go

@@ -14,6 +14,8 @@ type Instance struct {
 }
 
 type Options struct {
+	// AutoExtractRootDomains is used to extract root domains from the input list of subdomains
+	AutoExtractRootDomains bool
 	// Domain is the domain specified for enumeration
 	Domains []string
 	// Retries is the number of retries for dns

pkg/massdns/process.go

@@ -21,6 +21,7 @@ import (
 	folderutil "github.com/projectdiscovery/utils/folder"
 	stringsutil "github.com/projectdiscovery/utils/strings"
 	"github.com/remeh/sizedwaitgroup"
+	"github.com/weppos/publicsuffix-go/publicsuffix"
 )
 
 // runs massdns binary with the specified options
@@ -116,6 +117,16 @@ func (instance *Instance) Run(ctx context.Context) error {
 		gologger.Info().Msgf("Massdns input parsing completed in %s\n", time.Since(now))
 	}
 
+	if instance.options.AutoExtractRootDomains {
+		gologger.Info().Msgf("Started extracting root domains\n")
+		now := time.Now()
+		err = instance.autoExtractRootDomains(shstore)
+		if err != nil {
+			return fmt.Errorf("could not extract root domains: %w", err)
+		}
+		gologger.Info().Msgf("Root domain extraction completed in %s\n", time.Since(now))
+	}
+
 	// Perform wildcard filtering only if domain name has been specified
 	if len(instance.options.Domains) > 0 {
 		gologger.Info().Msgf("Started removing wildcards records\n")
@@ -186,6 +197,31 @@ func (instance *Instance) parseMassDNSOutputDir(tmpDir string, store *store.Stor
 	return nil
 }
 
+func (instance *Instance) autoExtractRootDomains(store *store.Store) error {
+	candidateRootDomains := make(map[string]struct{})
+	store.Iterate(func(ip string, hostnames []string, counter int) {
+		for _, hostname := range hostnames {
+			rootDomain, err := publicsuffix.Domain(hostname)
+			if err != nil {
+				continue
+			}
+			candidateRootDomains[rootDomain] = struct{}{}
+		}
+	})
+
+	// add the existing ones
+	for _, domain := range instance.options.Domains {
+		candidateRootDomains[domain] = struct{}{}
+	}
+
+	instance.options.Domains = make([]string, 0)
+	for item := range candidateRootDomains {
+		instance.options.Domains = append(instance.options.Domains, item)
+	}
+
+	return nil
+}
+
 func (instance *Instance) filterWildcards(st *store.Store) error {
 	// Start to work in parallel on wildcards
 	wildcardWg := sizedwaitgroup.New(instance.options.WildcardsThreads)
@@ -217,7 +253,6 @@ func (instance *Instance) filterWildcards(st *store.Store) error {
 					}
 
 					isWildcard, ips := instance.wildcardResolver.LookupHost(hostname)
-					gologger.Debug().Msgf("isWildcard: %v, ips: %v, hostname: %s\n", isWildcard, ips, hostname)
 					if len(ips) > 0 {
 						for ip := range ips {
 							// we add the single ip to the wildcard list

pkg/runner/options.go

@@ -12,28 +12,29 @@ import (
 // Options contains the configuration options for tuning
 // the active dns resolving process.
 type Options struct {
-	Directory          string              // Directory is a directory for temporary data
-	Domains            goflags.StringSlice // Domains is the list of domains to find subdomains
-	SubdomainsList     string              // SubdomainsList is the file containing list of hosts to resolve
-	ResolversFile      string              // ResolversFile is the file containing resolvers to use for enumeration
-	TrustedResolvers   string              // TrustedResolvers is the file containing trusted resolvers
-	Wordlist           string              // Wordlist is a wordlist to use for enumeration
-	MassdnsPath        string              // MassdnsPath contains the path to massdns binary
-	Output             string              // Output is the file to write found subdomains to.
-	Json               bool                // Json is the format for making output as ndjson
-	Silent             bool                // Silent suppresses any extra text and only writes found host:port to screen
-	Version            bool                // Version specifies if we should just show version and exit
-	Retries            int                 // Retries is the number of retries for dns enumeration
-	Verbose            bool                // Verbose flag indicates whether to show verbose output or not
-	NoColor            bool                // No-Color disables the colored output
-	Threads            int                 // Thread controls the number of parallel host to enumerate
-	MassdnsRaw         string              // MassdnsRaw perform wildcards filtering from an existing massdns output file
-	WildcardThreads    int                 // WildcardsThreads controls the number of parallel host to check for wildcard
-	StrictWildcard     bool                // StrictWildcard flag indicates whether wildcard check has to be performed on each found subdomains
-	WildcardOutputFile string              // StrictWildcard flag indicates whether wildcard check has to be performed on each found subdomains
-	MassDnsCmd         string              // Supports massdns flags(example -i)
-	DisableUpdateCheck bool                // DisableUpdateCheck disable automatic update check
-	Mode               string
+	AutoExtractRootDomains bool                // Automatically extract root domains
+	Directory              string              // Directory is a directory for temporary data
+	Domains                goflags.StringSlice // Domains is the list of domains to find subdomains
+	SubdomainsList         string              // SubdomainsList is the file containing list of hosts to resolve
+	ResolversFile          string              // ResolversFile is the file containing resolvers to use for enumeration
+	TrustedResolvers       string              // TrustedResolvers is the file containing trusted resolvers
+	Wordlist               string              // Wordlist is a wordlist to use for enumeration
+	MassdnsPath            string              // MassdnsPath contains the path to massdns binary
+	Output                 string              // Output is the file to write found subdomains to.
+	Json                   bool                // Json is the format for making output as ndjson
+	Silent                 bool                // Silent suppresses any extra text and only writes found host:port to screen
+	Version                bool                // Version specifies if we should just show version and exit
+	Retries                int                 // Retries is the number of retries for dns enumeration
+	Verbose                bool                // Verbose flag indicates whether to show verbose output or not
+	NoColor                bool                // No-Color disables the colored output
+	Threads                int                 // Thread controls the number of parallel host to enumerate
+	MassdnsRaw             string              // MassdnsRaw perform wildcards filtering from an existing massdns output file
+	WildcardThreads        int                 // WildcardsThreads controls the number of parallel host to check for wildcard
+	StrictWildcard         bool                // StrictWildcard flag indicates whether wildcard check has to be performed on each found subdomains
+	WildcardOutputFile     string              // StrictWildcard flag indicates whether wildcard check has to be performed on each found subdomains
+	MassDnsCmd             string              // Supports massdns flags(example -i)
+	DisableUpdateCheck     bool                // DisableUpdateCheck disable automatic update check
+	Mode                   string
 
 	OnResult func(*retryabledns.DNSData)
 }
@@ -53,6 +54,7 @@ func ParseOptions() *Options {
 
 	flagSet.CreateGroup("input", "Input",
 		flagSet.StringSliceVarP(&options.Domains, "domain", "d", nil, "Domain to find or resolve subdomains for", goflags.FileCommaSeparatedStringSliceOptions),
+		flagSet.BoolVarP(&options.AutoExtractRootDomains, "auto-domain", "ad", false, "Automatically extract root domains"),
 		flagSet.StringVarP(&options.SubdomainsList, "list", "l", "", "File containing list of subdomains to resolve"),
 		flagSet.StringVarP(&options.Wordlist, "wordlist", "w", "", "File containing words to bruteforce for domain"),
 		flagSet.StringVarP(&options.ResolversFile, "resolver", "r", "", "File containing list of resolvers for enumeration"),
@@ -95,7 +97,9 @@ func ParseOptions() *Options {
 		flagSet.BoolVarP(&options.NoColor, "no-color", "nc", false, "Don't Use colors in output"),
 	)
 
-	_ = flagSet.Parse()
+	if err := flagSet.Parse(); err != nil {
+		gologger.Fatal().Msgf("Program exiting: %s\n", err)
+	}
 
 	// Read the inputs and configure the logging
 	options.configureOutput()

pkg/runner/runner.go

@@ -169,22 +169,23 @@ func (r *Runner) processSubdomains() {
 // runMassdns runs the massdns tool on the list of inputs
 func (r *Runner) runMassdns(inputFile string) {
 	massdns, err := massdns.New(massdns.Options{
-		Domains:            r.options.Domains,
-		Retries:            r.options.Retries,
-		MassdnsPath:        r.options.MassdnsPath,
-		Threads:            r.options.Threads,
-		WildcardsThreads:   r.options.WildcardThreads,
-		InputFile:          inputFile,
-		ResolversFile:      r.options.ResolversFile,
-		TrustedResolvers:   r.options.TrustedResolvers,
-		TempDir:            r.tempDir,
-		OutputFile:         r.options.Output,
-		Json:               r.options.Json,
-		MassdnsRaw:         r.options.MassdnsRaw,
-		StrictWildcard:     r.options.StrictWildcard,
-		WildcardOutputFile: r.options.WildcardOutputFile,
-		MassDnsCmd:         r.options.MassDnsCmd,
-		OnResult:           r.options.OnResult,
+		Domains:                r.options.Domains,
+		AutoExtractRootDomains: r.options.AutoExtractRootDomains,
+		Retries:                r.options.Retries,
+		MassdnsPath:            r.options.MassdnsPath,
+		Threads:                r.options.Threads,
+		WildcardsThreads:       r.options.WildcardThreads,
+		InputFile:              inputFile,
+		ResolversFile:          r.options.ResolversFile,
+		TrustedResolvers:       r.options.TrustedResolvers,
+		TempDir:                r.tempDir,
+		OutputFile:             r.options.Output,
+		Json:                   r.options.Json,
+		MassdnsRaw:             r.options.MassdnsRaw,
+		StrictWildcard:         r.options.StrictWildcard,
+		WildcardOutputFile:     r.options.WildcardOutputFile,
+		MassDnsCmd:             r.options.MassDnsCmd,
+		OnResult:               r.options.OnResult,
 	})
 	if err != nil {
 		gologger.Error().Msgf("Could not create massdns client: %s\n", err)

pkg/wildcards/resolver.go

@@ -1,26 +1,30 @@
 package wildcards
 
 import (
+	"errors"
 	"fmt"
 	"strings"
 
 	"github.com/miekg/dns"
 	"github.com/projectdiscovery/dnsx/libs/dnsx"
 	"github.com/projectdiscovery/gologger"
+	sliceutil "github.com/projectdiscovery/utils/slice"
 	stringsutil "github.com/projectdiscovery/utils/strings"
 	"github.com/rs/xid"
 )
 
 // Resolver represents a dns resolver for removing wildcards
 type Resolver struct {
-	domains []string
+	Domains *sliceutil.SyncSlice[string]
 	client  *dnsx.DNSX
 }
 
 // NewResolver initializes and creates a new resolver to find wildcards
 func NewResolver(domains []string, retries int, resolvers []string) (*Resolver, error) {
+	fqdns := sliceutil.NewSyncSlice[string]()
+	fqdns.Append(domains...)
 	resolver := &Resolver{
-		domains: domains,
+		Domains: fqdns,
 	}
 
 	options := dnsx.DefaultOptions
@@ -44,12 +48,14 @@ func (w *Resolver) LookupHost(host string) (bool, map[string]struct{}) {
 	wildcards := make(map[string]struct{})
 
 	var domain string
-	for _, domainCandidate := range w.domains {
+	w.Domains.Each(func(i int, domainCandidate string) error {
 		if stringsutil.HasSuffixAny(host, "."+domainCandidate) {
 			domain = domainCandidate
-			break
+			// just to interrupt the iteration
+			return errors.New("found domain")
 		}
-	}
+		return nil
+	})
 
 	// ignore records without domain (todo: might be interesting to detect dangling domains)
 	if domain == "" {

tests/subdomains.txt

@@ -0,0 +1,4 @@
+aa.scanme.sh
+bb.scanme.sh
+cc.scanme.sh
+www.scanme.sh