首次提交

Makefile

@@ -0,0 +1,8 @@
+VERSION=$(shell git describe --tags --always)
+
+.PHONY: build_all
+build_all:
+	rm -rf bin && mkdir bin bin/linux-amd64 bin/darwin-amd64 bin/windows-amd64 \
+	&& GOOS=darwin GOARCH=amd64 go build -ldflags "-X main.Version=$(VERSION)" -o ./bin/darwin-amd64/ ./cmd/flamingo \
+	&& GOOS=linux GOARCH=amd64 go build -ldflags "-X main.Version=$(VERSION)" -o ./bin/linux-amd64/ ./cmd/flamingo \
+	&& GOOS=windows GOARCH=amd64 go build -ldflags "-X main.Version=$(VERSION)" -o ./bin/windows-amd64/ ./cmd/flamingo

README.md

@@ -0,0 +1,66 @@
+## flamingo
+
+**flamingo** 是一个开源的浏览器爬虫工具，用于收集 HTTP 请求对象。将这些请求对象提供给漏洞扫描器，以帮助检测 Web 漏洞。
+
+### 特性
+
+1.  驱动 Headless Chrome，搭建原生浏览器爬虫环境；
+2.  遍历 DOM 节点，获取页面中静态链接，包括注释中的链接；
+3.  使用 Hook 技术收集 DOM 0级和 DOM 2级事件，并自动化触发；
+4.  监控 DOM 变化，发现动态产生的链接；
+5.  遍历表单节点，自动化填充和提交表单。
+
+### 安装
+
+**安装使用之前，请务必阅读并同意 [免责声明](./disclaimer.md) 中的条款，否则请勿安装使用本工具。**
+
+#### 编译
+
+```bash
+$ make build_all
+```
+
+在 Linux 或 Darwin 平台上运行，请赋予二进制程序可执行权限。
+
+#### 运行
+
+```bash
+$ ./flamingo -h
+
+NAME:
+   flamingo - A browser crawler for web vulnerability scanner
+
+USAGE:
+   flamingo [global options] url
+
+VERSION:
+   0.0.1
+
+AUTHOR:
+   yns0ng <[email protected]>
+
+COMMANDS:
+   help, h  Shows a list of commands or help for one command
+
+GLOBAL OPTIONS:
+   --config file, -c file  Load options from a config file
+   --output-json file      custom output json file path, saved full request dump
+   --log-level value       custom log level (debug|info|warn|error|fatal)
+   --gui                   display browser GUI (default: false)
+   --help, -h              show help
+   --version, -v           print the version
+```
+
+### 使用
+
+使用 flamingo 前，请先下载 [Chromium]((https://www.chromium.org/getting-involved/download-chromium)) 可执行程序，并配置文件 flamingo.yml 中的配置项 **chromium_path**。
+
+flamingo 默认从同级目录的 flamingo.yml 文件中加载配置项，也可以使用 **--config** 选项指定任意路径的配置文件。
+
+```bash
+$ ./flamingo --output-json testphp.json http://testphp.vulnweb.com/
+```
+
+运行结果截图：
+
+![demo](./demo.jpg)

cmd/flamingo/main.go

@@ -0,0 +1,186 @@
+package main
+
+import (
+	b64 "encoding/base64"
+	"encoding/json"
+	"io/ioutil"
+	"log"
+	"os"
+	"os/signal"
+	"path/filepath"
+	"syscall"
+	"time"
+
+	"github.com/phplaber/flamingo/config"
+	"github.com/phplaber/flamingo/pkg"
+	"github.com/urfave/cli/v2"
+	"gopkg.in/yaml.v2"
+)
+
+type cmdOptions struct {
+	configFile string
+	jsonFile   string
+	logLevel   string
+	gui        bool
+}
+
+type FileOptions struct {
+	ExtraHeaders            map[string]interface{} `yaml:"extra_headers"`
+	TabExecuteTimeout       int                    `yaml:"tab_execute_timeout"`
+	DomLoadTimeout          int                    `yaml:"dom_load_timeout"`
+	DomLoadAndHandleTimeout int                    `yaml:"dom_load_and_handle_timeout"`
+	MaxWorkerNum            int                    `yaml:"max_worker_num"`
+	Proxy                   string                 `yaml:"proxy"`
+	ChromiumPath            string                 `yaml:"chromium_path"`
+	MaxPageVisitNum         int                    `yaml:"max_page_visit_num"`
+}
+
+type options struct {
+	cmd  cmdOptions
+	file FileOptions
+}
+
+var conf options
+
+func run(c *cli.Context) error {
+	start := time.Now()
+	// 初始化日志程序
+	log := pkg.InitLogger(conf.cmd.logLevel)
+
+	// 处理各种中断程序执行信号
+	s := make(chan os.Signal, 1)
+	signal.Notify(s, syscall.SIGTERM, syscall.SIGHUP, syscall.SIGINT)
+	go func() {
+		<-s
+		log.Info("program was interrupted")
+		os.Exit(0)
+	}()
+
+	// 检查必选参数
+	url := c.Args().Get(0)
+	if url == "" {
+		log.Fatal("target URL parameter is required")
+	}
+
+	// 解析配置文件选项
+	pwd, _ := os.Getwd()
+	configFile := filepath.Join(pwd, "flamingo.yml")
+	if conf.cmd.configFile != "" {
+		configFile = filepath.Join(pwd, conf.cmd.configFile)
+	}
+
+	// 读文件
+	confData, err := ioutil.ReadFile(configFile)
+	if err != nil {
+		log.Fatal("read config file error:", err)
+	}
+	// 解析配置项
+	if err := yaml.Unmarshal(confData, &conf.file); err != nil {
+		log.Fatal("parse yaml error:", err)
+	}
+
+	// 构造初始请求对象
+	request := &pkg.Request{
+		URL:     pkg.ParseURL(url),
+		Method:  config.HTTPMethod["get"],
+		Headers: conf.file.ExtraHeaders,
+		Data:    "",
+		Source:  config.LinkSource["fromTarget"],
+	}
+
+	// 初始化浏览器
+	browser := &pkg.Browser{
+		Headless:     !conf.cmd.gui,
+		ExtraHeaders: conf.file.ExtraHeaders,
+		Proxy:        conf.file.Proxy,
+		ChromiumPath: conf.file.ChromiumPath,
+	}
+	pkg.TabExecuteTimeout = time.Duration(conf.file.TabExecuteTimeout) * time.Second
+	pkg.DomLoadTimeout = time.Duration(conf.file.DomLoadTimeout) * time.Second
+	browser.Init()
+	defer browser.Close()
+
+	// 初始化 worker 池
+	pool := &pkg.Pool{
+		Capacity:        conf.file.MaxWorkerNum,
+		MaxPageVisitNum: conf.file.MaxPageVisitNum,
+	}
+	pool.Init()
+	defer pool.Close()
+
+	// 开始执行
+	pool.Run(&pkg.Task{
+		Pool:    pool,
+		Browser: browser,
+		Request: request,
+	})
+
+	// 输出结果到 JSON 文件
+	var requestList []map[string]interface{}
+	for _, requestResult := range pkg.ResultAll.AllResult {
+		request := make(map[string]interface{})
+		request["Method"] = requestResult.Method
+		request["URL"] = requestResult.URL.String()
+		request["Header"] = requestResult.Headers
+		request["b64_body"] = b64.StdEncoding.EncodeToString([]byte(requestResult.Data))
+		requestList = append(requestList, request)
+	}
+
+	reqBytes, err := json.Marshal(requestList)
+	if err != nil {
+		log.Fatal("marshal request list error:", err.Error())
+	}
+
+	pkg.WriteFile(conf.cmd.jsonFile, reqBytes)
+
+	log.Infof("Crawling finished, %d requests crawled in %s.", len(requestList), time.Since(start))
+	return nil
+}
+
+// 版本号 编译时赋值
+var Version string
+
+func main() {
+	author := cli.Author{
+		Name:  "yns0ng",
+		Email: "[email protected]",
+	}
+
+	app := &cli.App{
+		Name:      "flamingo",
+		Usage:     "A browser crawler for web vulnerability scanner",
+		UsageText: "flamingo [global options] url",
+		Version:   Version,
+		Authors:   []*cli.Author{&author},
+		Flags: []cli.Flag{
+			&cli.StringFlag{
+				Name:        "config",
+				Aliases:     []string{"c"},
+				Usage:       "Load options from a config `file`",
+				Destination: &conf.cmd.configFile,
+			},
+			&cli.StringFlag{
+				Name:        "output-json",
+				Usage:       "custom output json `file` path, saved full request dump",
+				Destination: &conf.cmd.jsonFile,
+				Required:    true,
+			},
+			&cli.StringFlag{
+				Name:        "log-level",
+				Usage:       "custom log level (debug|info|warn|error|fatal)",
+				Destination: &conf.cmd.logLevel,
+			},
+			&cli.BoolFlag{
+				Name:        "gui",
+				Value:       false,
+				Usage:       "display browser GUI",
+				Destination: &conf.cmd.gui,
+			},
+		},
+		Action: run,
+	}
+
+	if err := app.Run(os.Args); err != nil {
+		log.Fatal(err)
+	}
+}