refactor project

.github/workflows/build.yml

@@ -10,28 +10,29 @@ jobs:
     strategy:
       matrix:
         os: [ubuntu-latest, macos-latest, windows-latest]
-        go: [1.18]
+        go: ["1.20"]
     name: ${{ matrix.os }} @ Go ${{ matrix.go }}
     runs-on: ${{ matrix.os }}
     steps:
       - name: Checkout code
-        uses: actions/checkout@v2
+        uses: actions/checkout@v3
         with:
           ref: main
 
       - name: Set up Go ${{ matrix.go }}
-        uses: actions/setup-go@v2
+        uses: actions/setup-go@v4
         with:
           go-version: ${{ matrix.go }}
 
       - name: Build
         run: go build -ldflags "-s -w" -trimpath ./cmd/sf
 
       - name: Upload artifact
-        uses: actions/upload-artifact@v2
+        uses: actions/upload-artifact@v3
         if: success()
         with:
-          name: bin@${{ matrix.os }}
+          name: ${{ matrix.os }}
+          if-no-files-found: ignore
           path: |
             sf.exe
             sf

.gitignore

@@ -1,101 +1,6 @@
-.idea/
 .vscode/
-sf-dev/
 *.txt
 *.pprof
-
-# Created by https://www.toptal.com/developers/gitignore/api/go,linux,macos,windows
-# Edit at https://www.toptal.com/developers/gitignore?templates=go,linux,macos,windows
-
-### Go ###
-# Binaries for programs and plugins
-*.exe
-*.exe~
-*.dll
-*.so
-*.dylib
-
-# Test binary, built with `go test -c`
-*.test
-
-# Output of the go coverage tool, specifically when used with LiteIDE
-*.out
-
-# Dependency directories (remove the comment below to include it)
-# vendor/
-
-### Go Patch ###
-/vendor/
-/Godeps/
-
-### Linux ###
-*~
-
-# temporary files which can be created if a process still has a handle open of a deleted file
-.fuse_hidden*
-
-# KDE directory preferences
-.directory
-
-# Linux trash folder which might appear on any partition or disk
-.Trash-*
-
-# .nfs files are created when an open file is removed but is still being accessed
-.nfs*
-
-### macOS ###
-# General
 .DS_Store
-.AppleDouble
-.LSOverride
-
-# Icon must end with two \r
-Icon
-
-
-# Thumbnails
 ._*
 
-# Files that might appear in the root of a volume
-.DocumentRevisions-V100
-.fseventsd
-.Spotlight-V100
-.TemporaryItems
-.Trashes
-.VolumeIcon.icns
-.com.apple.timemachine.donotpresent
-
-# Directories potentially created on remote AFP share
-.AppleDB
-.AppleDesktop
-Network Trash Folder
-Temporary Items
-.apdisk
-
-### Windows ###
-# Windows thumbnail cache files
-Thumbs.db
-Thumbs.db:encryptable
-ehthumbs.db
-ehthumbs_vista.db
-
-# Dump file
-*.stackdump
-
-# Folder config file
-[Dd]esktop.ini
-
-# Recycle Bin used on file shares
-$RECYCLE.BIN/
-
-# Windows Installer files
-*.cab
-*.msi
-*.msix
-*.msm
-*.msp
-
-# Windows shortcuts
-*.lnk
-
-# End of https://www.toptal.com/developers/gitignore/api/go,linux,macos,windows

README.md

@@ -4,40 +4,24 @@
 [![Go Report Card](https://goreportcard.com/badge/github.com/0x2E/sf)](https://goreportcard.com/report/github.com/0x2E/sf)
 [![go version](https://img.shields.io/github/go-mod/go-version/0x2E/sf)](https://github.com/0x2E/sf/blob/main/go.mod)
 
-SF 是一个高效的子域名收集工具。目前已有字典爆破、域传送模块。
-
-<details>
-    <summary>演示</summary>
-    <a href="https://asciinema.org/a/447397" target="_blank"><img src="https://asciinema.org/a/447397.svg" /></a>
-</details>
-
-子域名由各个模块收集后送入任务队列，按需进行域名解析（Enumerator）、有效性检测（Checker）、记录（Recorder）。
-
-Enumerator 基于 UDP 的无状态特性，将发送和接收分离，效率更高，支持限流和重试机制。
-
-Checker 根据 DNS 记录等特征筛选有效子域名，目前用于泛解析检测（[关于泛解析检测的一些问题](https://github.com/0x2E/sf/issues/12)）。
+SF 是一个高效的子域名爆破工具：
 
+- 基于 UDP 的无连接特性并行收发 DNS 请求
+- 支持检测域传送漏洞
+- 支持秒级限流和失败重试
+- 支持基于 `*.` 记录的泛解析域名检测
 
 ## 安装
 
-- 使用编译好的可执行文件
-  - 稳定：[release](https://github.com/0x2E/sf/releases)
-  - 主分支最新：进入 [Actions](https://github.com/0x2E/sf/actions) 中任意一次 workflow，下滑页面找到 Artifacts
-- 编译源码
-
-## 使用方法
+1. 编译好的可执行文件
 
-- `-u` 目标域名（必需）
-- `-f` 字典路径，为空时不启动爆破模块
-- `-r` DNS 服务器，默认 `8.8.8.8`
-- `-o` 结果输出路径，默认 `{domain}.txt`
-- `-check` 是否开启有效性检查，默认 true
-- `-t` 并发数，默认 200
-- `-rate` 每秒最大请求量，默认 2000
-- `-retry` 重试次数，默认 3
-- `-h` 输出完整参数列表
+   - [Release](https://github.com/0x2E/sf/releases)
+   - 主分支的最新提交：[Actions](https://github.com/0x2E/sf/actions) 中任意一次 workflow 的 Artifacts
 
-## TODO
+2. 编译源码
 
-- [开发计划](https://github.com/0x2E/sf/labels/todo)
-- [提出建议](https://github.com/0x2E/sf/issues/new)
+```shell
+git clone https://github.com/0x2E/sf.git
+cd sf
+go build -o sf ./cmd/sf
+```

cmd/sf/banner.go

@@ -0,0 +1,13 @@
+package main
+
+const banner = `
+   .-'''-.   ________
+  / _     \ |        |
+ ( ' )/---' |   .----'
+(_ o _).    |  _|____
+ (_,_).  \  |_( )_   |
+.---.  \  | (_ o._)__|
+\    \ -  . |(_,_)
+ \       /  |   |
+   -...-'   '---'
+`

cmd/sf/sf.go

@@ -2,61 +2,91 @@ package main
 
 import (
 	"bufio"
-	"flag"
-	"github.com/0x2E/sf/internal/conf"
-	"github.com/0x2E/sf/internal/engine"
-	"log"
+	"fmt"
 	"net/http"
 	_ "net/http/pprof"
 	"os"
 	"strings"
 	"time"
-)
 
-const (
-	DEBUG = false
+	flag "github.com/spf13/pflag"
+
+	"github.com/0x2E/sf/internal/conf"
+	"github.com/0x2E/sf/internal/engine"
+	"github.com/sirupsen/logrus"
 )
 
 func main() {
-	debug()
-
-	var output string
-	c := &conf.Config{}
-	flag.StringVar(&c.Domain, "u", "", "Target domain name or URL")
-	flag.StringVar(&c.Wordlist, "f", "", "Load wordlist from a file")
-	flag.StringVar(&c.Resolver, "r", engine.RESOLVER, "DNS resolver")
-	flag.StringVar(&output, "o", "", "Output results to a file")
-	flag.IntVar(&c.Thread, "t", engine.THREAD, "Number of concurrent")
-	flag.IntVar(&c.Rate, "rate", engine.RATE, "Maximum number of DNS requests sent per second")
-	flag.IntVar(&c.Retry, "retry", engine.RETRY, "Number of retries")
-	flag.BoolVar(&c.Check, "check", engine.CHECK, "Whether to check the validity of the subdomain, set to false with '-check=false'")
+	var (
+		c      = conf.C
+		output string
+		slient bool
+		debug  bool
+		help   bool
+	)
+	flag.StringVarP(&c.Target, "domain", "d", "", "Target domain name")
+	flag.StringVarP(&c.Wordlist, "wordlist", "w", "", "Wordlist file")
+	flag.StringVarP(&c.Resolver, "resolver", "r", "8.8.8.8", "DNS resolver")
+	flag.StringVarP(&output, "output", "o", "", "Output results to a file")
+	flag.IntVarP(&c.Concurrent, "concurrent", "t", 200, "Number of concurrent")
+	flag.IntVar(&c.Rate, "rate", 1000, `Maximum rate DNS req/s. 
+It is recommended to determine if the rate is appropriate by the send/recv statistics in log`)
+	flag.IntVar(&c.Retry, "retry", 1, "Number of retries")
+	flag.IntVarP(&c.StatisticsInterval, "stats", "s", 2, "Statistics interval(seconds) in log")
+	flag.BoolVar(&c.ValidCheck, "check", false, "Check the validity of the subdomains")
+	flag.BoolVar(&slient, "slient", false, "Only output valid subdomains, and logs that caused abnormal exit, e.g., fatal and panic")
+	flag.BoolVar(&debug, "debug", false, "Set the log level to debug, and enable golang pprof with web service")
+	flag.BoolVarP(&help, "help", "h", false, "Show help message")
+	flag.CommandLine.SortFlags = false
 	flag.Parse()
 
-	if err := c.Verify(); err != nil {
-		log.Fatal(err)
+	if help {
+		flag.Usage()
+		os.Exit(0)
 	}
 
-	if strings.TrimSpace(output) == "" {
-		output = c.Domain + "txt" // domain结尾已经有一个点了
+	logrus.SetFormatter(&logrus.TextFormatter{
+		TimestampFormat: "20060102 15:04:05",
+		FullTimestamp:   true,
+	})
+	if slient {
+		if debug {
+			logrus.Fatal("cannot enable 'debug' and 'slient' at the same time")
+		}
+		logrus.SetLevel(logrus.FatalLevel)
+	} else {
+		fmt.Print(banner)
 	}
 
-	startTime := time.Now()
+	if debug {
+		logrus.SetLevel(logrus.DebugLevel)
+		go pprof()
+	}
+
+	if err := c.Verify(); err != nil {
+		logrus.Fatal(err)
+	}
 
+	logrus.Infof("target: [%s]. wordlist: [%s]. resolver: [%s]. concurrent: [%d]. rate: [%d]. retry: [%d]. check valid: [%t]",
+		c.Target, c.Wordlist, c.Resolver, c.Concurrent, c.Rate, c.Retry, c.ValidCheck)
+
+	startAt := time.Now()
 	app := engine.New(c)
 	valid, invalid := app.Run()
-	log.Printf("Found %d valid, %d invalid. %.2f seconds in total.\n", len(valid), len(invalid), time.Since(startTime).Seconds())
+
+	logrus.Infof("found %d valid, %d invalid. %.2f seconds in total.\n", len(valid), len(invalid), time.Since(startAt).Seconds())
 
 	saveResult(output, valid)
-	saveResult("invalid_"+output, invalid)
 }
 
 func saveResult(path string, data []string) {
-	if len(data) == 0 {
+	if strings.TrimSpace(path) == "" || len(data) == 0 {
 		return
 	}
-	f, err := os.OpenFile(path, os.O_CREATE|os.O_TRUNC|os.O_RDWR, 0666)
+
+	f, err := os.OpenFile(path, os.O_CREATE|os.O_TRUNC|os.O_RDWR, 0o755)
 	if err != nil {
-		log.Printf("cannot save results into file: %s", err)
+		logrus.Error("cannot save results:", err)
 		return
 	}
 	defer f.Close()
@@ -65,22 +95,11 @@ func saveResult(path string, data []string) {
 		bufWriter.WriteString(v + "\n")
 	}
 	bufWriter.Flush()
-	log.Printf("Results are stored in %s\n", path)
 }
 
-func debug() {
-	if DEBUG {
-		//f, _ := os.OpenFile("sf-dev/cpu.pprof", os.O_CREATE|os.O_TRUNC|os.O_RDWR, 0644)
-		//defer f.Close()
-		//pprof.StartCPUProfile(f)
-		//defer pprof.StopCPUProfile()
-
-		go func() {
-			// localhost:10010/debug/pprof
-			if err := http.ListenAndServe(":10010", nil); err != nil {
-				log.Fatal(err)
-			}
-			os.Exit(0)
-		}()
+func pprof() {
+	logrus.Debug("pprof is on 0.0.0.0:10000/debug/pprof")
+	if err := http.ListenAndServe(":10000", nil); err != nil {
+		logrus.Error(err)
 	}
 }

go.mod

@@ -1,23 +1,19 @@
 module github.com/0x2E/sf
 
-go 1.18
+go 1.20
 
 require (
-	github.com/miekg/dns v1.1.50
+	github.com/miekg/dns v1.1.55
 	github.com/pkg/errors v0.9.1
-	github.com/schollz/progressbar/v3 v3.13.0
-	go.uber.org/ratelimit v0.2.0
+	github.com/sirupsen/logrus v1.9.3
+	github.com/spf13/pflag v1.0.5
+	go.uber.org/ratelimit v0.3.0
 )
 
 require (
-	github.com/andres-erbsen/clock v0.0.0-20160526145045-9e14626cd129 // indirect
-	github.com/mattn/go-runewidth v0.0.14 // indirect
-	github.com/mitchellh/colorstring v0.0.0-20190213212951-d06e56a500db // indirect
-	github.com/rivo/uniseg v0.4.3 // indirect
-	golang.org/x/mod v0.4.2 // indirect
-	golang.org/x/net v0.0.0-20210726213435-c6fcb2dbf985 // indirect
-	golang.org/x/sys v0.4.0 // indirect
-	golang.org/x/term v0.4.0 // indirect
-	golang.org/x/tools v0.1.6-0.20210726203631-07bc1bf47fb2 // indirect
-	golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1 // indirect
+	github.com/benbjohnson/clock v1.3.0 // indirect
+	golang.org/x/mod v0.7.0 // indirect
+	golang.org/x/net v0.2.0 // indirect
+	golang.org/x/sys v0.2.0 // indirect
+	golang.org/x/tools v0.3.0 // indirect
 )