{{ $info.Result }}
{{ end }} diff --git a/conf/file.go b/conf/file.go index 53ce06a..779d618 100644 --- a/conf/file.go +++ b/conf/file.go @@ -178,6 +178,7 @@ collection: - host - href - redirect + - referer - u - ip - address @@ -232,9 +233,12 @@ collection: - k8s - docker - env + - ak + - sk - _key # 这种以 _ 开头的会不完全匹配，包含 _key 就会抛出来 - _token - _secret + - _uri `) // HotConf 使用 viper 对配置热加载 diff --git a/go.mod b/go.mod index c3d54f0..0d8ab51 100644 --- a/go.mod +++ b/go.mod @@ -1,8 +1,6 @@ module github.com/yhy0/Jie -go 1.22.0 - -toolchain go1.22.1 +go 1.22.3 require ( github.com/PuerkitoBio/goquery v1.9.2 @@ -79,7 +77,7 @@ require ( github.com/stretchr/testify v1.9.0 github.com/tidwall/match v1.1.1 github.com/tomatome/grdp v0.1.1-0.20230622130233-b1e80faa1cf2 - github.com/wasilibs/go-re2 v1.5.2 + github.com/wasilibs/go-re2 v1.5.3 github.com/weppos/publicsuffix-go v0.30.2 github.com/yaklang/yaklang v1.3.2 github.com/yhy0/logging v0.0.0-20231128014545-22711cccc3b0 diff --git a/go.sum b/go.sum index a2b06b5..91971ce 100644 --- a/go.sum +++ b/go.sum @@ -1249,8 +1249,8 @@ github.com/valyala/bytebufferpool v1.0.0 h1:GqA5TC/0021Y/b9FG4Oi9Mr3q7XYx6Kllzaw github.com/valyala/bytebufferpool v1.0.0/go.mod h1:6bBcMArwyJ5K/AmCkWv1jt77kVWyCJ6HpOuEn7z0Csc= github.com/valyala/fasttemplate v1.2.2 h1:lxLXG0uE3Qnshl9QyaK6XJxMXlQZELvChBOCmQD0Loo= github.com/valyala/fasttemplate v1.2.2/go.mod h1:KHLXt3tVN2HBp8eijSv/kGJopbvo7S+qRAEEKiv+SiQ= -github.com/wasilibs/go-re2 v1.5.2 h1:fDO2TJrRzRrv3jD0gzOvmZ2UM4Yt9YXOEdLrlNc/Ies= -github.com/wasilibs/go-re2 v1.5.2/go.mod h1:UqqxQ1O99boQUm1r61H/IYGiGQOS/P88K7hU5nLNkEg= +github.com/wasilibs/go-re2 v1.5.3 h1:wiuTcgDZdLhu8NG8oqF5sF5Q3yIU14lPAvXqeYzDK3g= +github.com/wasilibs/go-re2 v1.5.3/go.mod h1:PzpVPsBdFC7vM8QJbbEnOeTmwA0DGE783d/Gex8eCV8= github.com/wasilibs/nottinygc v0.4.0 h1:h1TJMihMC4neN6Zq+WKpLxgd9xCFMw7O9ETLwY2exJQ= github.com/wasilibs/nottinygc v0.4.0/go.mod h1:oDcIotskuYNMpqMF23l7Z8uzD4TC0WXHK8jetlB3HIo= github.com/weppos/publicsuffix-go v0.12.0/go.mod h1:z3LCPQ38eedDQSwmsSRW4Y7t2L8Ln16JPQ02lHAdn5k= diff --git a/lib/cdncheck/other_test.go b/lib/cdncheck/other_test.go index 6345d2b..0dab481 100644 --- a/lib/cdncheck/other_test.go +++ b/lib/cdncheck/other_test.go @@ -2,19 +2,19 @@ package cdncheck import ( "testing" - + "github.com/projectdiscovery/retryabledns" "github.com/stretchr/testify/require" ) func TestCheckSuffix(t *testing.T) { client := New() - + valid, provider, _, err := client.CheckSuffix("test.cloudfront.net") require.Nil(t, err, "could not check cname") require.True(t, valid, "could not get valid cname") require.Equal(t, "amazon", provider, "could not get correct provider") - + valid, _, _, err = client.CheckSuffix("test.provider.net") require.Nil(t, err, "could not check cname") require.False(t, valid, "could get valid cname") @@ -22,17 +22,17 @@ func TestCheckSuffix(t *testing.T) { func TestCheckWappalyzer(t *testing.T) { client := New() - + valid, provider, err := client.CheckWappalyzer(map[string]struct{}{"imperva": {}}) require.Nil(t, err, "could not check wappalyzer") require.True(t, valid, "could not get valid cname") require.Equal(t, "imperva", provider, "could not get correct provider") - + valid, provider, err = client.CheckWappalyzer(map[string]struct{}{"imperva:4.5.6": {}}) require.Nil(t, err, "could not check wappalyzer") require.True(t, valid, "could not get valid cname") require.Equal(t, "imperva", provider, "could not get correct provider") - + valid, _, err = client.CheckWappalyzer(map[string]struct{}{"php": {}}) require.Nil(t, err, "could not check cname") require.False(t, valid, "could get valid cname") @@ -40,8 +40,8 @@ func TestCheckWappalyzer(t *testing.T) { func TestCheckDomainWithFallback(t *testing.T) { client := New() - - valid, provider, itemType, err := client.CheckDomainWithFallback("www.gap.com") + + valid, provider, itemType, err, _ := client.CheckDomainWithFallback("www.gap.com") require.Nil(t, err, "could not check") require.True(t, valid, "could not check domain") require.Equal(t, "akamai", provider, "could not get correct provider") @@ -54,20 +54,20 @@ func TestCheckDNSResponse(t *testing.T) { defaultMaxRetries := 3 retryabledns, _ := retryabledns.New(defaultResolvers, defaultMaxRetries) dnsData, _ := retryabledns.Resolve("hackerone.com") - - valid, provider, itemType, err := client.CheckDNSResponse(dnsData) - + + valid, provider, itemType, err, _ := client.CheckDNSResponse(dnsData) + require.Nil(t, err, "could not check cname") require.True(t, valid, "could not get valid cname") require.Equal(t, "cloudflare", provider, "could not get correct provider") require.Equal(t, "waf", itemType, "could not get correct itemType") - + dnsData, _ = retryabledns.CNAME("www.gap.com") - - valid, provider, itemType, err = client.CheckDNSResponse(dnsData) + + valid, provider, itemType, err, _ = client.CheckDNSResponse(dnsData) require.Nil(t, err, "could not check") require.True(t, valid, "could not check domain") require.Equal(t, "akamai", provider, "could not get correct provider") require.Equal(t, "waf", itemType, "could not get correct itemType") - + } diff --git a/pkg/mitmproxy/go-mitmproxy.go b/pkg/mitmproxy/go-mitmproxy.go index 4f115a3..660e9b4 100644 --- a/pkg/mitmproxy/go-mitmproxy.go +++ b/pkg/mitmproxy/go-mitmproxy.go @@ -9,9 +9,11 @@ package mitmproxy import ( "github.com/panjf2000/ants/v2" "github.com/yhy0/Jie/conf" + "github.com/yhy0/Jie/pkg/mitmproxy/go-mitmproxy/helper" "github.com/yhy0/Jie/pkg/mitmproxy/go-mitmproxy/proxy" "github.com/yhy0/Jie/pkg/task" "github.com/yhy0/logging" + "net/http" ) var t *task.Task @@ -46,6 +48,18 @@ func NewMitmproxy() { logging.Logger.Fatal(err) } + // 直接从这里限制走不走代理，之前那种方式也会走代理，只不过不会经过扫描流程 + if len(conf.GlobalConfig.Mitmproxy.Exclude) > 0 || !(len(conf.GlobalConfig.Mitmproxy.Exclude) == 1 && conf.GlobalConfig.Mitmproxy.Exclude[0] == "") { + PassiveProxy.SetShouldInterceptRule(func(req *http.Request) bool { + return !helper.MatchHost(req.Host, conf.GlobalConfig.Mitmproxy.Exclude) + }) + } + if len(conf.GlobalConfig.Mitmproxy.Include) > 0 && !(len(conf.GlobalConfig.Mitmproxy.Include) == 1 && conf.GlobalConfig.Mitmproxy.Include[0] == "") { + PassiveProxy.SetShouldInterceptRule(func(req *http.Request) bool { + return helper.MatchHost(req.Host, conf.GlobalConfig.Mitmproxy.Include) + }) + } + // 添加一个插件用来获取流量信息 PassiveProxy.AddAddon(&PassiveAddon{}) go func() { diff --git a/pkg/task/task.go b/pkg/task/task.go index e4a400b..42b93da 100644 --- a/pkg/task/task.go +++ b/pkg/task/task.go @@ -254,10 +254,9 @@ func (t *Task) Distribution(in *input.CrawlResult) DistributionTaskFunc { if ok { output.SCopilotMessage[in.Host].CollectionMsg.Parameters.Set(_para, v.(int)+1) } else { - output.SCopilotMessage[in.Host].CollectionMsg.Parameters.Set(_para, 0) + output.SCopilotMessage[in.Host].CollectionMsg.Parameters.Set(_para, 1) } } - // 按照value的字典序升序排序 output.SCopilotMessage[in.Host].CollectionMsg.Parameters.Sort(func(a *orderedmap.Pair, b *orderedmap.Pair) bool { return a.Value().(int) > b.Value().(int) diff --git a/test/Jie_config.yaml b/test/Jie_config.yaml deleted file mode 100644 index 25d10c4..0000000 --- a/test/Jie_config.yaml +++ /dev/null @@ -1,217 +0,0 @@ -version: 1.1.1 - -parallel: 10 # 同时扫描的最大 url 个数 - -# 全局 http 发包配置 -http: - proxy: "" # 漏洞扫描时使用的代理，如: http://127.0.0.1:8080 - timeout: 10 # 建立 tcp 连接的超时时间 - maxConnsPerHost: 100 # 每个 host 最大连接数 - retryTimes: 0 # 请求失败的重试次数，0 则不重试 - allowRedirect: 0 # 单个请求最大允许的跳转数，0 则不跳转 - verifySSL: false # 是否验证 ssl 证书 - maxQps: 50 # 每秒最大请求数 - headers: # 指定 http 请求头 - forceHTTP1: false # 强制指定使用 http/1.1, 不然会根据服务器选择，如果服务器支持 http2，默认会使用 http2 - -# 漏洞探测的插件配置 -plugins: - bruteForce: - web: false # web 服务类的爆破，比如 tomcat 爆破 - service: false # 服务类的爆破，比如 mysql 爆破 - usernameDict: "" # 自定义用户名字典, 为空将使用内置字典, 配置后将与内置字典**合并** - passwordDict: "" # 自定义密码字典，为空将使用内置字典, 配置后将与内置字典**合并** - cmdInjection: - enabled: true - crlfInjection: - enabled: true - xss: - enabled: true - detectXssInCookie: true # 是否探测入口点在 cookie 中的 xss - sql: - enabled: true - booleanBasedDetection: true # 是否检测布尔盲注 - errorBasedDetection: true # 是否检测报错注入 - timeBasedDetection: true # 是否检测时间盲注 - detectInCookie: true # 是否检查在 cookie 中的注入 - sqlmapApi: - enabled: false - url: "" # sqlmap api 的地址 - username: "" # 认证用户名 - password: "" # 认证密码 - xxe: - enabled: true - ssrf: - enabled: true - bbscan: # bbscan https://github.com/lijiejie/bbscan 这种规则类目录扫描 - enabled: true - jsonp: - enabled: true - log4j: - enabled: true - bypass403: - enabled: true - fastjson: - enabled: true - archive: # 从 https://web.archive.org/ 获取历史 url，作为补充扫描 - enabled: true - iis: # iis 短文件名 fuzz - enabled: false - nginxAliasTraversal: # nginx 别名遍历 - enabled: true - poc: - enabled: false - nuclei: - enabled: false - portScan: - enabled: false - -# 反连平台配置 -# 注意: 默认配置为 dig.pm, 可以使用 https://github.com/yumusb/DNSLog-Platform-Golang 自行搭建，后续看需求要不要支持别的 dnslog 平台 -reverse: - host: "https://dig.pm/" # 反连平台地址 - Domain: "ipv6.bypass.eu.org." # 指定反连域名 - -# 基础爬虫配置 这里都没写呢，后边看看要不要写一下 -basicCrawler: - maxDepth: 0 # 最大爬取深度， 0 为无限制 - maxCountOfLinks: 0 # 本次爬取收集的最大链接数, 0 为无限制 - allowVisitParentPath: false # 是否允许爬取父目录, 如果扫描目标为 t.com/a/且该项为 false, 那么就不会爬取 t.com/ 这级的内容 - restriction: # 爬虫的允许爬取的资源限制, 为空表示不限制。爬虫会自动添加扫描目标到 Hostname_allowed。 - hostname_allowed: [] # 允许访问的 Hostname，支持格式如 t.com、*.t.com、1.1.1.1、1.1.1.1/24、1.1-4.1.1-8 - hostname_disallowed: # 不允许访问的 Hostname，支持格式如 t.com、*.t.com、1.1.1.1、1.1.1.1/24、1.1-4.1.1-8 - - '*.edu.*' - - '*.gov.*' - port_allowed: [] # 允许访问的端口, 支持的格式如: 80、80-85 - port_disallowed: [] # 不允许访问的端口, 支持的格式如: 80、80-85 - path_allowed: [] # 允许访问的路径，支持的格式如: test、*test* - path_disallowed: [] # 不允许访问的路径, 支持的格式如: test、*test* - query_key_allowed: [] # 允许访问的 Query Key，支持的格式如: test、*test* - query_key_disallowed: [] # 不允许访问的 Query Key, 支持的格式如: test、*test* - fragment_allowed: [] # 允许访问的 Fragment, 支持的格式如: test、*test* - fragment_disallowed: [] # 不允许访问的 Fragment, 支持的格式如: test、*test* - post_key_allowed: [] # 允许访问的 Post Body 中的参数, 支持的格式如: test、*test* - post_key_disallowed: [] # 不允许访问的 Post Body 中的参数, 支持的格式如: test、*test* - basic_auth: # 基础认证信息 - username: "" - password: "" - -# 被动代理配置 -mitmproxy: - caCert: ./ca.crt # CA 根证书路径 - caKey: ./ca.key # CA 私钥路径 - basicAuth: # 基础认证的用户名密码 - header: "Go-Mitmproxy-Authorization" # 认证头 - username: "" - password: "" - exclude: # 不允许访问的 Hostname，支持格式如 t.com、*.t.com、 todo 1.1.1.1、1.1.1.1/24、1.1-4.1.1-8 - - .google. - - .googleapis. - - .gstatic. - - .googleusercontent. - - .googlevideo. - - .firefox. - - .firefoxchina.cn - - .firefoxusercontent.com - - .mozilla. - - .doubleclick. - - spocs.getpocket.com - - .portswigger.net - - .gov.(com|cn) - - cdn.jsdelivr.net - - cdn-go.cn - include: # 允许访问的 Hostname，支持格式如 t.com、*.t.com、1.1.1.1、1.1.1.1/24、1.1-4.1.1-8 - - - # 排除的后缀, 不会被扫描器扫描 按格式增加 - filterSuffix: .3g2, .3gp, .7z, .apk, .arj, .avi, .axd, .bmp, .csv, .deb, .dll, .doc, .drv, .eot, .exe, .flv, .gif, .gifv, .gz, .h264, .ico, .iso, .jar, .jpeg, .jpg, .lock, .m4a, .m4v, .map, .mkv, .mov, .mp3, .mp4, .mpeg, .mpg, .msi, .ogg, .ogm, .ogv, .otf, .pdf, .pkg, .png, .ppt, .psd, .rar, .rm, .rpm, .svg, .swf, .sys, .tar.gz, .tar, .tif, .tiff, .ttf, .txt, .vob, .wav, .webm, .webp, .wmv, .woff, .woff2, .xcf, .xls, .xlsx, .zip - maxLength: 3000 # 队列长度限制, 也可以理解为最大允许多少等待扫描的请求, 请根据内存大小自行调整，这个还没有实现，我没有使用队列 - -# 信息收集类的正则 -collection: - domain: - - "['\"](([a-zA-Z0-9]{1,9}:)?//)?(.{1,36}:.{1,36}@)?[a-zA-Z0-9\\-\\.]*?\\.(xin|com|cn|net|com\\.cn|vip|top|cc|shop|club|wang|xyz|luxe|site|news|pub|fun|online|win|red|loan|ren|mom|net\\.cn|org|link|biz|bid|help|tech|date|mobi|so|me|tv|co|vc|pw|video|party|pics|website|store|ltd|ink|trade|live|wiki|space|gift|lol|work|band|info|click|photo|market|tel|social|press|game|kim|org\\.cn|games|pro|men|love|studio|rocks|asia|group|science|design|software|engineer|lawyer|fit|beer|我爱你|中国|公司|网络|在线|网址|网店|集团|中文网)(:\\d{1,5})?" - ip: - - "['\"](([a-zA-Z0-9]{1,9}:)?//)?(.{1,36}:.{1,36}@)?\\d{1,3}\\.\\d{1,3}\\.\\d{1,3}\\.\\d{1,3}(:\\d{1,5})?" - phone: - - "['\"](1(3([0-35-9]\\d|4[1-8])|4[14-9]\\d|5([\\d]\\d|7[1-79])|66\\d|7[2-35-8]\\d|8\\d{2}|9[89]\\d)\\d{7})['\"]" - email: - - "['\"]([\\w!#$%&'*+=?^_`{|}~-]+(?:\\.[\\w!#$%&'*+=?^_`{|}~-]+)*@(?:[\\w](?:[\\w-]*[\\w])?\\.)+[\\w](?:[\\w-]*[\\w])?)['\"]" - api: # 自己来写正则吧，网上找的都不太靠谱, 见到了慢慢补吧 - - "(?i)\\.(get|post|put|delete|options|connect|trace|patch)\\([\"'](/?.*?)[\"']" - - "(?:\"|')(/[^/\"']+){2,}(?:\"|')" - url: - - "[\"'‘“`]\\s{0,6}(https{0,1}:[-a-zA-Z0-9()@:%_\\+.~#?&//={}]{2,250}?)\\s{0,6}[\"'‘“`]" - - "=\\s{0,6}(https{0,1}:[-a-zA-Z0-9()@:%_\\+.~#?&//={}]{2,250})" - - "[\"'‘“`]\\s{0,6}([#,.]{0,2}/[-a-zA-Z0-9()@:%_\\+.~#?&//={}]{2,250}?)\\s{0,6}[\"'‘“`]" - - "\"([-a-zA-Z0-9()@:%_\\+.~#?&//={}]+?[/]{1}[-a-zA-Z0-9()@:%_\\+.~#?&//={}]+?)\"" - - "href\\s{0,6}=\\s{0,6}[\"'‘“`]{0,1}\\s{0,6}([-a-zA-Z0-9()@:%_\\+.~#?&//={}]{2,250})|action\\s{0,6}=\\s{0,6}[\"'‘“`]{0,1}\\s{0,6}([-a-zA-Z0-9()@:%_\\+.~#?&//={}]{2,250})" - urlFilter: - - "\\.js\\?|\\.css\\?|\\.jpeg\\?|\\.jpg\\?|\\.png\\?|.gif\\?|www\\.w3\\.org|example\\.com|\\<|\\>|\\{|\\}|\\[|\\]|\\||\\^|;|/js/|\\.src|\\.replace|\\.url|\\.att|\\.href|location\\.href|javascript:|location:|text/.*?|application/.*?|\\.createObject|:location|\\.path|\\*#__PURE__\\*|\\*\\$0\\*|\\n" - - ".*\\.js$|.*\\.css$|.*\\.scss$|.*,$|.*\\.jpeg$|.*\\.jpg$|.*\\.png$|.*\\.gif$|.*\\.ico$|.*\\.svg$|.*\\.vue$|.*\\.ts$" - idCard: - - "['\"]((\\d{8}(0\\d|10|11|12)([0-2]\\d|30|31)\\d{3}$)|(\\d{6}(18|19|20)\\d{2}(0[1-9]|10|11|12)([0-2]\\d|30|31)\\d{3}(\\d|X|x)))['\"]" - other: - - "(access.{0,1}key|access.{0,1}Key|access.{0,1}Id|access.{0,1}id|.{0,8}密码|.{0,8}账号|默认.{0,8}|加密|解密|(password|pwd|pass|username|user|name|account):\\s+[\"'].{1,36}['\"])" - - "['\"](ey[A-Za-z0-9_-]{10,}\\.[A-Za-z0-9._-]{10,}|ey[A-Za-z0-9_\\/+-]{10,}\\.[A-Za-z0-9._\\/+-]{10,})['\"]" - sensitiveParameters: # 请求或者回显中一些可能可以利用的参数 不区分大小写 - - url - - host - - href - - redirect - - u - - ip - - address - - addr - - file - - f - - dir - - directory - - path - - router - - callback - - conf - - cfg - - config - - jdbc - - db - - sql - - api - - apikey - - api_key - - access - - key - - token - - access_token - - accessToken - - stable_token - - authorizer - - authorizer_access_token - - authorizerAccessToken - - appid - - appSecret - - app_secret - - secret - - auth - - oauth - - oauth2 - - corp - - admin - - pass - - pwd - - passwd - - password - - debug - - dbg - - exe - - exec - - execute - - load - - shell - - grant - - create - - k8s - - docker - - env - - _key # 这种以 _ 开头的会不完全匹配，包含 _key 就会抛出来 - - _token - - _secret