feat 一些优化;re2 更新

SCopilot/templates/SCopilot.html

@@ -174,7 +174,7 @@ <h4 class="card-title">端口信息</h4>
 
                     <ul class="list-group position-relative">
                         {{ range $index, $info := .data.InfoMsg }}
-                            <li class="list-group-item msg-plugin-ul" data-plugin="{{ $info.Plugin }}">
+                            <li class="list-group-item msg-plugin-ul" data-plugin="{{ $info.Plugin }}" style="margin-top:10px;">
                                 {{ $info.Url }}
                                 <span class="badge rounded-pill bg-primary"> {{ $info.Plugin }}</span>
                                 {{ if $info.Result }}<p><span class="badge bg-warning">{{ $info.Result }}</span></p>{{ end }}

conf/file.go

@@ -178,6 +178,7 @@ collection:
     - host
     - href
     - redirect
+    - referer
     - u
     - ip
     - address
@@ -232,9 +233,12 @@ collection:
     - k8s
     - docker
     - env
+    - ak
+    - sk
     - _key  # 这种以 _ 开头的会不完全匹配，包含 _key 就会抛出来
     - _token
     - _secret
+    - _uri
 `)
 
 // HotConf 使用 viper 对配置热加载

go.mod

@@ -1,8 +1,6 @@
 module github.com/yhy0/Jie
 
-go 1.22.0
-
-toolchain go1.22.1
+go 1.22.3
 
 require (
 	github.com/PuerkitoBio/goquery v1.9.2
@@ -79,7 +77,7 @@ require (
 	github.com/stretchr/testify v1.9.0
 	github.com/tidwall/match v1.1.1
 	github.com/tomatome/grdp v0.1.1-0.20230622130233-b1e80faa1cf2
-	github.com/wasilibs/go-re2 v1.5.2
+	github.com/wasilibs/go-re2 v1.5.3
 	github.com/weppos/publicsuffix-go v0.30.2
 	github.com/yaklang/yaklang v1.3.2
 	github.com/yhy0/logging v0.0.0-20231128014545-22711cccc3b0

go.sum

@@ -1249,8 +1249,8 @@ github.com/valyala/bytebufferpool v1.0.0 h1:GqA5TC/0021Y/b9FG4Oi9Mr3q7XYx6Kllzaw
 github.com/valyala/bytebufferpool v1.0.0/go.mod h1:6bBcMArwyJ5K/AmCkWv1jt77kVWyCJ6HpOuEn7z0Csc=
 github.com/valyala/fasttemplate v1.2.2 h1:lxLXG0uE3Qnshl9QyaK6XJxMXlQZELvChBOCmQD0Loo=
 github.com/valyala/fasttemplate v1.2.2/go.mod h1:KHLXt3tVN2HBp8eijSv/kGJopbvo7S+qRAEEKiv+SiQ=
-github.com/wasilibs/go-re2 v1.5.2 h1:fDO2TJrRzRrv3jD0gzOvmZ2UM4Yt9YXOEdLrlNc/Ies=
-github.com/wasilibs/go-re2 v1.5.2/go.mod h1:UqqxQ1O99boQUm1r61H/IYGiGQOS/P88K7hU5nLNkEg=
+github.com/wasilibs/go-re2 v1.5.3 h1:wiuTcgDZdLhu8NG8oqF5sF5Q3yIU14lPAvXqeYzDK3g=
+github.com/wasilibs/go-re2 v1.5.3/go.mod h1:PzpVPsBdFC7vM8QJbbEnOeTmwA0DGE783d/Gex8eCV8=
 github.com/wasilibs/nottinygc v0.4.0 h1:h1TJMihMC4neN6Zq+WKpLxgd9xCFMw7O9ETLwY2exJQ=
 github.com/wasilibs/nottinygc v0.4.0/go.mod h1:oDcIotskuYNMpqMF23l7Z8uzD4TC0WXHK8jetlB3HIo=
 github.com/weppos/publicsuffix-go v0.12.0/go.mod h1:z3LCPQ38eedDQSwmsSRW4Y7t2L8Ln16JPQ02lHAdn5k=

lib/cdncheck/other_test.go

@@ -2,46 +2,46 @@ package cdncheck
 
 import (
     "testing"
-
+    
     "github.com/projectdiscovery/retryabledns"
     "github.com/stretchr/testify/require"
 )
 
 func TestCheckSuffix(t *testing.T) {
     client := New()
-
+    
     valid, provider, _, err := client.CheckSuffix("test.cloudfront.net")
     require.Nil(t, err, "could not check cname")
     require.True(t, valid, "could not get valid cname")
     require.Equal(t, "amazon", provider, "could not get correct provider")
-
+    
     valid, _, _, err = client.CheckSuffix("test.provider.net")
     require.Nil(t, err, "could not check cname")
     require.False(t, valid, "could get valid cname")
 }
 
 func TestCheckWappalyzer(t *testing.T) {
     client := New()
-
+    
     valid, provider, err := client.CheckWappalyzer(map[string]struct{}{"imperva": {}})
     require.Nil(t, err, "could not check wappalyzer")
     require.True(t, valid, "could not get valid cname")
     require.Equal(t, "imperva", provider, "could not get correct provider")
-
+    
     valid, provider, err = client.CheckWappalyzer(map[string]struct{}{"imperva:4.5.6": {}})
     require.Nil(t, err, "could not check wappalyzer")
     require.True(t, valid, "could not get valid cname")
     require.Equal(t, "imperva", provider, "could not get correct provider")
-
+    
     valid, _, err = client.CheckWappalyzer(map[string]struct{}{"php": {}})
     require.Nil(t, err, "could not check cname")
     require.False(t, valid, "could get valid cname")
 }
 
 func TestCheckDomainWithFallback(t *testing.T) {
     client := New()
-
-    valid, provider, itemType, err := client.CheckDomainWithFallback("www.gap.com")
+    
+    valid, provider, itemType, err, _ := client.CheckDomainWithFallback("www.gap.com")
     require.Nil(t, err, "could not check")
     require.True(t, valid, "could not check domain")
     require.Equal(t, "akamai", provider, "could not get correct provider")
@@ -54,20 +54,20 @@ func TestCheckDNSResponse(t *testing.T) {
     defaultMaxRetries := 3
     retryabledns, _ := retryabledns.New(defaultResolvers, defaultMaxRetries)
     dnsData, _ := retryabledns.Resolve("hackerone.com")
-
-    valid, provider, itemType, err := client.CheckDNSResponse(dnsData)
-
+    
+    valid, provider, itemType, err, _ := client.CheckDNSResponse(dnsData)
+    
     require.Nil(t, err, "could not check cname")
     require.True(t, valid, "could not get valid cname")
     require.Equal(t, "cloudflare", provider, "could not get correct provider")
     require.Equal(t, "waf", itemType, "could not get correct itemType")
-
+    
     dnsData, _ = retryabledns.CNAME("www.gap.com")
-
-    valid, provider, itemType, err = client.CheckDNSResponse(dnsData)
+    
+    valid, provider, itemType, err, _ = client.CheckDNSResponse(dnsData)
     require.Nil(t, err, "could not check")
     require.True(t, valid, "could not check domain")
     require.Equal(t, "akamai", provider, "could not get correct provider")
     require.Equal(t, "waf", itemType, "could not get correct itemType")
-
+    
 }

pkg/mitmproxy/go-mitmproxy.go

@@ -9,9 +9,11 @@ package mitmproxy
 import (
     "github.com/panjf2000/ants/v2"
     "github.com/yhy0/Jie/conf"
+    "github.com/yhy0/Jie/pkg/mitmproxy/go-mitmproxy/helper"
     "github.com/yhy0/Jie/pkg/mitmproxy/go-mitmproxy/proxy"
     "github.com/yhy0/Jie/pkg/task"
     "github.com/yhy0/logging"
+    "net/http"
 )
 
 var t *task.Task
@@ -46,6 +48,18 @@ func NewMitmproxy() {
         logging.Logger.Fatal(err)
     }
 
+    // 直接从这里限制走不走代理，之前那种方式也会走代理，只不过不会经过扫描流程
+    if len(conf.GlobalConfig.Mitmproxy.Exclude) > 0 || !(len(conf.GlobalConfig.Mitmproxy.Exclude) == 1 && conf.GlobalConfig.Mitmproxy.Exclude[0] == "") {
+        PassiveProxy.SetShouldInterceptRule(func(req *http.Request) bool {
+            return !helper.MatchHost(req.Host, conf.GlobalConfig.Mitmproxy.Exclude)
+        })
+    }
+    if len(conf.GlobalConfig.Mitmproxy.Include) > 0 && !(len(conf.GlobalConfig.Mitmproxy.Include) == 1 && conf.GlobalConfig.Mitmproxy.Include[0] == "") {
+        PassiveProxy.SetShouldInterceptRule(func(req *http.Request) bool {
+            return helper.MatchHost(req.Host, conf.GlobalConfig.Mitmproxy.Include)
+        })
+    }
+
     // 添加一个插件用来获取流量信息
     PassiveProxy.AddAddon(&PassiveAddon{})
     go func() {

pkg/task/task.go

@@ -254,10 +254,9 @@ func (t *Task) Distribution(in *input.CrawlResult) DistributionTaskFunc {
                     if ok {
                         output.SCopilotMessage[in.Host].CollectionMsg.Parameters.Set(_para, v.(int)+1)
                     } else {
-                        output.SCopilotMessage[in.Host].CollectionMsg.Parameters.Set(_para, 0)
+                        output.SCopilotMessage[in.Host].CollectionMsg.Parameters.Set(_para, 1)
                     }
                 }
-
                 // 按照value的字典序升序排序
                 output.SCopilotMessage[in.Host].CollectionMsg.Parameters.Sort(func(a *orderedmap.Pair, b *orderedmap.Pair) bool {
                     return a.Value().(int) > b.Value().(int)