From af9cbec832e907a4065b228bfb1083e3a376d330 Mon Sep 17 00:00:00 2001
From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com>
Date: Mon, 13 May 2024 05:40:59 +0000
Subject: [PATCH 001/283] chore(deps): bump
github.com/projectdiscovery/retryabledns
Bumps [github.com/projectdiscovery/retryabledns](https://github.com/projectdiscovery/retryabledns) from 1.0.58 to 1.0.59.
- [Release notes](https://github.com/projectdiscovery/retryabledns/releases)
- [Commits](https://github.com/projectdiscovery/retryabledns/compare/v1.0.58...v1.0.59)
---
updated-dependencies:
- dependency-name: github.com/projectdiscovery/retryabledns
dependency-type: direct:production
update-type: version-update:semver-patch
...
Signed-off-by: dependabot[bot]
---
go.mod | 2 +-
go.sum | 4 ++--
2 files changed, 3 insertions(+), 3 deletions(-)
diff --git a/go.mod b/go.mod
index 31992dc6a2..6a2eeb4f20 100644
--- a/go.mod
+++ b/go.mod
@@ -24,7 +24,7 @@ require (
github.com/projectdiscovery/hmap v0.0.41
github.com/projectdiscovery/interactsh v1.1.9
github.com/projectdiscovery/rawhttp v0.1.47
- github.com/projectdiscovery/retryabledns v1.0.58
+ github.com/projectdiscovery/retryabledns v1.0.59
github.com/projectdiscovery/retryablehttp-go v1.0.58
github.com/projectdiscovery/yamldoc-go v1.0.4
github.com/remeh/sizedwaitgroup v1.0.0
diff --git a/go.sum b/go.sum
index 78bd017040..fe64290d48 100644
--- a/go.sum
+++ b/go.sum
@@ -875,8 +875,8 @@ github.com/projectdiscovery/rawhttp v0.1.47 h1:bjhZBK+7iuvcu0QTRJjdS4wP747+33Sd5
github.com/projectdiscovery/rawhttp v0.1.47/go.mod h1:2XA7xODWEGoHpEB8qzgqsV8yVL3cg5G63ZaT9ALwU4g=
github.com/projectdiscovery/rdap v0.9.1-0.20221108103045-9865884d1917 h1:m03X4gBVSorSzvmm0bFa7gDV4QNSOWPL/fgZ4kTXBxk=
github.com/projectdiscovery/rdap v0.9.1-0.20221108103045-9865884d1917/go.mod h1:JxXtZC9e195awe7EynrcnBJmFoad/BNDzW9mzFkK8Sg=
-github.com/projectdiscovery/retryabledns v1.0.58 h1:ut1FSB9+GZ6zQIlKJFLqIz2RZs81EmkbsHTuIrWfYLE=
-github.com/projectdiscovery/retryabledns v1.0.58/go.mod h1:RobmKoNBgngAVE4H9REQtaLP1pa4TCyypHy1MWHT1mY=
+github.com/projectdiscovery/retryabledns v1.0.59 h1:8pMN+VibEBp29RIUior9LXUbx0RsBTjPC0008t2hfGU=
+github.com/projectdiscovery/retryabledns v1.0.59/go.mod h1:CwyQLDt9oqNIO/2ArALhAnUHJjZYdvJRSfGERRNPtoQ=
github.com/projectdiscovery/retryablehttp-go v1.0.58 h1:i5BlSJGgNnoTULyqLSe3d39o/XShxK4oEvx0e/gb9N4=
github.com/projectdiscovery/retryablehttp-go v1.0.58/go.mod h1:bbok7sSEplSwZOY91UlLdVilhavYos1RaCJLJx761V0=
github.com/projectdiscovery/sarif v0.0.1 h1:C2Tyj0SGOKbCLgHrx83vaE6YkzXEVrMXYRGLkKCr/us=
From 2800cb6744e09836517f28d54552938d1c4fa993 Mon Sep 17 00:00:00 2001
From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com>
Date: Mon, 13 May 2024 05:41:05 +0000
Subject: [PATCH 002/283] chore(deps): bump
github.com/projectdiscovery/wappalyzergo
Bumps [github.com/projectdiscovery/wappalyzergo](https://github.com/projectdiscovery/wappalyzergo) from 0.0.120 to 0.0.122.
- [Release notes](https://github.com/projectdiscovery/wappalyzergo/releases)
- [Commits](https://github.com/projectdiscovery/wappalyzergo/compare/v0.0.120...v0.0.122)
---
updated-dependencies:
- dependency-name: github.com/projectdiscovery/wappalyzergo
dependency-type: direct:production
update-type: version-update:semver-patch
...
Signed-off-by: dependabot[bot]
---
go.mod | 2 +-
go.sum | 4 ++--
2 files changed, 3 insertions(+), 3 deletions(-)
diff --git a/go.mod b/go.mod
index 31992dc6a2..17660b4e65 100644
--- a/go.mod
+++ b/go.mod
@@ -95,7 +95,7 @@ require (
github.com/projectdiscovery/uncover v1.0.7
github.com/projectdiscovery/useragent v0.0.49
github.com/projectdiscovery/utils v0.0.92
- github.com/projectdiscovery/wappalyzergo v0.0.120
+ github.com/projectdiscovery/wappalyzergo v0.0.122
github.com/redis/go-redis/v9 v9.1.0
github.com/seh-msft/burpxml v1.0.1
github.com/stretchr/testify v1.9.0
diff --git a/go.sum b/go.sum
index 78bd017040..7f46ae808f 100644
--- a/go.sum
+++ b/go.sum
@@ -891,8 +891,8 @@ github.com/projectdiscovery/useragent v0.0.49 h1:wQc9i+Xy+mUMJ45Ralv1JsQImRWqEOE
github.com/projectdiscovery/useragent v0.0.49/go.mod h1:jQz6X/usiXrPYE6B/1uVKuzIrBJXgw9hLC9eeNy38+0=
github.com/projectdiscovery/utils v0.0.92 h1:lGCmjUJhzoNX4FQZWpp80058pRlD0/dYxLJOSs07EqY=
github.com/projectdiscovery/utils v0.0.92/go.mod h1:d5uvD5qcRiK3qxZbBy9eatCqrCSuj9SObL04w/WgXSg=
-github.com/projectdiscovery/wappalyzergo v0.0.120 h1:dphOXnaT3rryo9h9fgbxnAVhtQ1uq61yyQZMYyHz960=
-github.com/projectdiscovery/wappalyzergo v0.0.120/go.mod h1:qW0PP+UBMcdQBBnwk+X6YYFs6huKNvn2BOVs4vQPru0=
+github.com/projectdiscovery/wappalyzergo v0.0.122 h1:xfNJ7VNzU/OGlgYtsyB5ppuOHdfWzU2B8cYATwTz54c=
+github.com/projectdiscovery/wappalyzergo v0.0.122/go.mod h1:qW0PP+UBMcdQBBnwk+X6YYFs6huKNvn2BOVs4vQPru0=
github.com/projectdiscovery/yamldoc-go v1.0.4 h1:eZoESapnMw6WAHiVgRwNqvbJEfNHEH148uthhFbG5jE=
github.com/projectdiscovery/yamldoc-go v1.0.4/go.mod h1:8PIPRcUD55UbtQdcfFR1hpIGRWG0P7alClXNGt1TBik=
github.com/prometheus/client_golang v0.9.1/go.mod h1:7SWBe2y4D6OKWSNQJUaRYU/AaXPKyh/dDVn+NZz0KFw=
From c2c0f0672f6554418bc5b85e98b78ed367594bad Mon Sep 17 00:00:00 2001
From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com>
Date: Mon, 13 May 2024 06:06:04 +0000
Subject: [PATCH 003/283] chore(deps): bump
github.com/projectdiscovery/retryablehttp-go
Bumps [github.com/projectdiscovery/retryablehttp-go](https://github.com/projectdiscovery/retryablehttp-go) from 1.0.58 to 1.0.60.
- [Release notes](https://github.com/projectdiscovery/retryablehttp-go/releases)
- [Commits](https://github.com/projectdiscovery/retryablehttp-go/compare/v1.0.58...v1.0.60)
---
updated-dependencies:
- dependency-name: github.com/projectdiscovery/retryablehttp-go
dependency-type: direct:production
update-type: version-update:semver-patch
...
Signed-off-by: dependabot[bot]
---
go.mod | 4 ++--
go.sum | 8 ++++----
2 files changed, 6 insertions(+), 6 deletions(-)
diff --git a/go.mod b/go.mod
index 6a2eeb4f20..4b9692a0d2 100644
--- a/go.mod
+++ b/go.mod
@@ -20,12 +20,12 @@ require (
github.com/olekukonko/tablewriter v0.0.5
github.com/pkg/errors v0.9.1
github.com/projectdiscovery/clistats v0.0.20
- github.com/projectdiscovery/fastdialer v0.0.69
+ github.com/projectdiscovery/fastdialer v0.0.70
github.com/projectdiscovery/hmap v0.0.41
github.com/projectdiscovery/interactsh v1.1.9
github.com/projectdiscovery/rawhttp v0.1.47
github.com/projectdiscovery/retryabledns v1.0.59
- github.com/projectdiscovery/retryablehttp-go v1.0.58
+ github.com/projectdiscovery/retryablehttp-go v1.0.60
github.com/projectdiscovery/yamldoc-go v1.0.4
github.com/remeh/sizedwaitgroup v1.0.0
github.com/rs/xid v1.5.0
diff --git a/go.sum b/go.sum
index fe64290d48..36745c65e6 100644
--- a/go.sum
+++ b/go.sum
@@ -837,8 +837,8 @@ github.com/projectdiscovery/clistats v0.0.20 h1:5jO5SLiRJ7f0nDV0ndBNmBeesbROouPo
github.com/projectdiscovery/clistats v0.0.20/go.mod h1:GJ2av0KnOvK0AISQnP8hyDclYIji1LVkx2l0pwnzAu4=
github.com/projectdiscovery/dsl v0.0.52 h1:jvIvF+qN8+MbI1MHtWJJKfWqAZQlCExL3ob7SddQbZE=
github.com/projectdiscovery/dsl v0.0.52/go.mod h1:xfcHwhy2HSaeGgh+1wqzOoCGm2XTdh5JzjBRBVHEMvI=
-github.com/projectdiscovery/fastdialer v0.0.69 h1:BfFQTyTB1hrw9sWCw4CjQfbmlpvnJCPZEmKtxcwJGbU=
-github.com/projectdiscovery/fastdialer v0.0.69/go.mod h1:RXrx7M2T3V3rMZ2h0X2/SsY93+RhgF/LmFa1E0MI3L8=
+github.com/projectdiscovery/fastdialer v0.0.70 h1:1rnUKc8NRj6dzG8aTmqW+RF8m0dFbfTs9CiOo6Aig8U=
+github.com/projectdiscovery/fastdialer v0.0.70/go.mod h1:HQ0ZpvOPOTZFSQxGyYJgNdek93hi4eIC1avZgiQ7+a4=
github.com/projectdiscovery/fasttemplate v0.0.2 h1:h2cISk5xDhlJEinlBQS6RRx0vOlOirB2y3Yu4PJzpiA=
github.com/projectdiscovery/fasttemplate v0.0.2/go.mod h1:XYWWVMxnItd+r0GbjA1GCsUopMw1/XusuQxdyAIHMCw=
github.com/projectdiscovery/freeport v0.0.5 h1:jnd3Oqsl4S8n0KuFkE5Hm8WGDP24ITBvmyw5pFTHS8Q=
@@ -877,8 +877,8 @@ github.com/projectdiscovery/rdap v0.9.1-0.20221108103045-9865884d1917 h1:m03X4gB
github.com/projectdiscovery/rdap v0.9.1-0.20221108103045-9865884d1917/go.mod h1:JxXtZC9e195awe7EynrcnBJmFoad/BNDzW9mzFkK8Sg=
github.com/projectdiscovery/retryabledns v1.0.59 h1:8pMN+VibEBp29RIUior9LXUbx0RsBTjPC0008t2hfGU=
github.com/projectdiscovery/retryabledns v1.0.59/go.mod h1:CwyQLDt9oqNIO/2ArALhAnUHJjZYdvJRSfGERRNPtoQ=
-github.com/projectdiscovery/retryablehttp-go v1.0.58 h1:i5BlSJGgNnoTULyqLSe3d39o/XShxK4oEvx0e/gb9N4=
-github.com/projectdiscovery/retryablehttp-go v1.0.58/go.mod h1:bbok7sSEplSwZOY91UlLdVilhavYos1RaCJLJx761V0=
+github.com/projectdiscovery/retryablehttp-go v1.0.60 h1:sXbx6Rdh22SZ3AFhY3P7LC+p8GPLlANMgPHlkBXJlv8=
+github.com/projectdiscovery/retryablehttp-go v1.0.60/go.mod h1:rgRdV7LSrrTTlvN7yKsYxtvWm39VZB6pgD2t1p1ma64=
github.com/projectdiscovery/sarif v0.0.1 h1:C2Tyj0SGOKbCLgHrx83vaE6YkzXEVrMXYRGLkKCr/us=
github.com/projectdiscovery/sarif v0.0.1/go.mod h1:cEYlDu8amcPf6b9dSakcz2nNnJsoz4aR6peERwV+wuQ=
github.com/projectdiscovery/stringsutil v0.0.2 h1:uzmw3IVLJSMW1kEg8eCStG/cGbYYZAja8BH3LqqJXMA=
From 16ce116fed5c3a6b4adc321e2f262276db69e4b2 Mon Sep 17 00:00:00 2001
From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com>
Date: Mon, 13 May 2024 06:31:25 +0000
Subject: [PATCH 004/283] chore(deps): bump
github.com/projectdiscovery/fastdialer
Bumps [github.com/projectdiscovery/fastdialer](https://github.com/projectdiscovery/fastdialer) from 0.0.69 to 0.0.71.
- [Release notes](https://github.com/projectdiscovery/fastdialer/releases)
- [Commits](https://github.com/projectdiscovery/fastdialer/compare/v0.0.69...v0.0.71)
---
updated-dependencies:
- dependency-name: github.com/projectdiscovery/fastdialer
dependency-type: direct:production
update-type: version-update:semver-patch
...
Signed-off-by: dependabot[bot]
---
go.mod | 2 +-
go.sum | 4 ++--
2 files changed, 3 insertions(+), 3 deletions(-)
diff --git a/go.mod b/go.mod
index a9dd94c74c..1002e41efd 100644
--- a/go.mod
+++ b/go.mod
@@ -20,7 +20,7 @@ require (
github.com/olekukonko/tablewriter v0.0.5
github.com/pkg/errors v0.9.1
github.com/projectdiscovery/clistats v0.0.20
- github.com/projectdiscovery/fastdialer v0.0.70
+ github.com/projectdiscovery/fastdialer v0.0.71
github.com/projectdiscovery/hmap v0.0.41
github.com/projectdiscovery/interactsh v1.1.9
github.com/projectdiscovery/rawhttp v0.1.47
diff --git a/go.sum b/go.sum
index 69ad9c9002..dbc2f68aeb 100644
--- a/go.sum
+++ b/go.sum
@@ -837,8 +837,8 @@ github.com/projectdiscovery/clistats v0.0.20 h1:5jO5SLiRJ7f0nDV0ndBNmBeesbROouPo
github.com/projectdiscovery/clistats v0.0.20/go.mod h1:GJ2av0KnOvK0AISQnP8hyDclYIji1LVkx2l0pwnzAu4=
github.com/projectdiscovery/dsl v0.0.52 h1:jvIvF+qN8+MbI1MHtWJJKfWqAZQlCExL3ob7SddQbZE=
github.com/projectdiscovery/dsl v0.0.52/go.mod h1:xfcHwhy2HSaeGgh+1wqzOoCGm2XTdh5JzjBRBVHEMvI=
-github.com/projectdiscovery/fastdialer v0.0.70 h1:1rnUKc8NRj6dzG8aTmqW+RF8m0dFbfTs9CiOo6Aig8U=
-github.com/projectdiscovery/fastdialer v0.0.70/go.mod h1:HQ0ZpvOPOTZFSQxGyYJgNdek93hi4eIC1avZgiQ7+a4=
+github.com/projectdiscovery/fastdialer v0.0.71 h1:96j6Y65hDPZ9AzlYpp95hvIH5Yx/0OE2UTx+frWfnm4=
+github.com/projectdiscovery/fastdialer v0.0.71/go.mod h1:b/oPPVSoLLD2N4W2/HrXbhQbyJVXqRw8CK1lenCUk64=
github.com/projectdiscovery/fasttemplate v0.0.2 h1:h2cISk5xDhlJEinlBQS6RRx0vOlOirB2y3Yu4PJzpiA=
github.com/projectdiscovery/fasttemplate v0.0.2/go.mod h1:XYWWVMxnItd+r0GbjA1GCsUopMw1/XusuQxdyAIHMCw=
github.com/projectdiscovery/freeport v0.0.5 h1:jnd3Oqsl4S8n0KuFkE5Hm8WGDP24ITBvmyw5pFTHS8Q=
From 9adfc531c7724ea9828bd23697faadc2d7d964f3 Mon Sep 17 00:00:00 2001
From: Mzack9999
Date: Wed, 15 May 2024 15:34:59 +0200
Subject: [PATCH 005/283] uniforming sizes with utils
---
cmd/integration-test/http.go | 3 ++-
cmd/nuclei/main.go | 3 ++-
go.mod | 4 ++--
go.sum | 8 ++++----
internal/pdcp/writer.go | 5 +++--
pkg/output/output.go | 3 ++-
pkg/protocols/common/automaticscan/automaticscan.go | 3 ++-
pkg/protocols/http/request.go | 11 ++++++-----
pkg/protocols/offlinehttp/request.go | 3 ++-
pkg/protocols/protocols.go | 3 ++-
pkg/reporting/format/format_utils.go | 3 ++-
pkg/testutils/testutils.go | 3 ++-
pkg/types/types.go | 5 +++--
pkg/utils/monitor/monitor.go | 3 ++-
14 files changed, 36 insertions(+), 24 deletions(-)
diff --git a/cmd/integration-test/http.go b/cmd/integration-test/http.go
index b4f957e692..5b218aa53b 100644
--- a/cmd/integration-test/http.go
+++ b/cmd/integration-test/http.go
@@ -23,6 +23,7 @@ import (
logutil "github.com/projectdiscovery/utils/log"
sliceutil "github.com/projectdiscovery/utils/slice"
stringsutil "github.com/projectdiscovery/utils/strings"
+ unitutils "github.com/projectdiscovery/utils/unit"
)
var httpTestcases = []TestCaseInfo{
@@ -509,7 +510,7 @@ func (h *httpPostMultipartBody) Execute(filePath string) error {
var routerErr error
router.POST("/", func(w http.ResponseWriter, r *http.Request, _ httprouter.Params) {
- if err := r.ParseMultipartForm(1 * 1024); err != nil {
+ if err := r.ParseMultipartForm(unitutils.Mega); err != nil {
routerErr = err
return
}
diff --git a/cmd/nuclei/main.go b/cmd/nuclei/main.go
index a1123b9ffc..8be0207797 100644
--- a/cmd/nuclei/main.go
+++ b/cmd/nuclei/main.go
@@ -38,6 +38,7 @@ import (
"github.com/projectdiscovery/nuclei/v3/pkg/utils/monitor"
errorutil "github.com/projectdiscovery/utils/errors"
fileutil "github.com/projectdiscovery/utils/file"
+ unitutils "github.com/projectdiscovery/utils/unit"
updateutils "github.com/projectdiscovery/utils/update"
)
@@ -301,7 +302,7 @@ on extensive configurability, massive extensibility and ease of use.`)
flagSet.StringVarP(&options.AttackType, "attack-type", "at", "", "type of payload combinations to perform (batteringram,pitchfork,clusterbomb)"),
flagSet.StringVarP(&options.SourceIP, "source-ip", "sip", "", "source ip address to use for network scan"),
flagSet.IntVarP(&options.ResponseReadSize, "response-size-read", "rsr", 0, "max response size to read in bytes"),
- flagSet.IntVarP(&options.ResponseSaveSize, "response-size-save", "rss", 1*1024*1024, "max response size to read in bytes"),
+ flagSet.IntVarP(&options.ResponseSaveSize, "response-size-save", "rss", unitutils.Mega, "max response size to read in bytes"),
flagSet.DurationVarP(&options.ResponseReadTimeout, "response-read-timeout", "rrt", time.Duration(5*time.Second), "response read timeout in seconds"),
flagSet.CallbackVar(resetCallback, "reset", "reset removes all nuclei configuration and data files (including nuclei-templates)"),
flagSet.BoolVarP(&options.TlsImpersonate, "tls-impersonate", "tlsi", false, "enable experimental client hello (ja3) tls randomization"),
diff --git a/go.mod b/go.mod
index 1002e41efd..39156d2f6d 100644
--- a/go.mod
+++ b/go.mod
@@ -21,7 +21,7 @@ require (
github.com/pkg/errors v0.9.1
github.com/projectdiscovery/clistats v0.0.20
github.com/projectdiscovery/fastdialer v0.0.71
- github.com/projectdiscovery/hmap v0.0.41
+ github.com/projectdiscovery/hmap v0.0.42
github.com/projectdiscovery/interactsh v1.1.9
github.com/projectdiscovery/rawhttp v0.1.47
github.com/projectdiscovery/retryabledns v1.0.59
@@ -94,7 +94,7 @@ require (
github.com/projectdiscovery/tlsx v1.1.6
github.com/projectdiscovery/uncover v1.0.7
github.com/projectdiscovery/useragent v0.0.49
- github.com/projectdiscovery/utils v0.0.92
+ github.com/projectdiscovery/utils v0.0.94-0.20240515132152-3908d4bb5d2c
github.com/projectdiscovery/wappalyzergo v0.0.122
github.com/redis/go-redis/v9 v9.1.0
github.com/seh-msft/burpxml v1.0.1
diff --git a/go.sum b/go.sum
index dbc2f68aeb..4a95dbd040 100644
--- a/go.sum
+++ b/go.sum
@@ -853,8 +853,8 @@ github.com/projectdiscovery/gostruct v0.0.2 h1:s8gP8ApugGM4go1pA+sVlPDXaWqNP5BBD
github.com/projectdiscovery/gostruct v0.0.2/go.mod h1:H86peL4HKwMXcQQtEa6lmC8FuD9XFt6gkNR0B/Mu5PE=
github.com/projectdiscovery/gozero v0.0.2 h1:8fJeaCjxL9tpm33uG/RsCQs6HGM/NE6eA3cjkilRQ+E=
github.com/projectdiscovery/gozero v0.0.2/go.mod h1:d8bZvDWW07LWNYWrwjZ4OO1I0cpkfqaysyDfSs9ibK8=
-github.com/projectdiscovery/hmap v0.0.41 h1:8IgTyDce3/2JzcfPVA4H+XpBRFfETULx8td3BMdSYVE=
-github.com/projectdiscovery/hmap v0.0.41/go.mod h1:bCrai6x5Eijqm2U+jtcH0wZX5ZcaZhcvzoMGTZgLAf0=
+github.com/projectdiscovery/hmap v0.0.42 h1:+P8CC7gAeTG0phe0d1FB7i3Vl15v1K+dJApwX4rvMAM=
+github.com/projectdiscovery/hmap v0.0.42/go.mod h1:lbGBuL/bLoYWdlgphZmHXjZCYzteVDf4WfKsR/aH57c=
github.com/projectdiscovery/httpx v1.6.0 h1:6g4UoSQpsOyZgaK+SMLLnZIAU0eYyTxBUwVl+jtm0JQ=
github.com/projectdiscovery/httpx v1.6.0/go.mod h1:dzMzOWKfeofaXcXzac3O+YmuY24P0CRnviKGxvol6MM=
github.com/projectdiscovery/interactsh v1.1.9 h1:b77SaSGrO+DtivmWwqGGY2dmNlQC3Zgmwlaj9L4Oqvc=
@@ -889,8 +889,8 @@ github.com/projectdiscovery/uncover v1.0.7 h1:ut+2lTuvmftmveqF5RTjMWAgyLj8ltPQC7
github.com/projectdiscovery/uncover v1.0.7/go.mod h1:HFXgm1sRPuoN0D4oATljPIdmbo/EEh1wVuxQqo/dwFE=
github.com/projectdiscovery/useragent v0.0.49 h1:wQc9i+Xy+mUMJ45Ralv1JsQImRWqEOEvpYUe6MchScg=
github.com/projectdiscovery/useragent v0.0.49/go.mod h1:jQz6X/usiXrPYE6B/1uVKuzIrBJXgw9hLC9eeNy38+0=
-github.com/projectdiscovery/utils v0.0.92 h1:lGCmjUJhzoNX4FQZWpp80058pRlD0/dYxLJOSs07EqY=
-github.com/projectdiscovery/utils v0.0.92/go.mod h1:d5uvD5qcRiK3qxZbBy9eatCqrCSuj9SObL04w/WgXSg=
+github.com/projectdiscovery/utils v0.0.94-0.20240515132152-3908d4bb5d2c h1:vGsulLeJNcFjQTp+aSAO7auX5wriKrLrZ8+SILxtRcM=
+github.com/projectdiscovery/utils v0.0.94-0.20240515132152-3908d4bb5d2c/go.mod h1:wxPi+kCsLm5JCLMkZJyGwS+4Mn4PaPHHf0ayE8JphOw=
github.com/projectdiscovery/wappalyzergo v0.0.122 h1:xfNJ7VNzU/OGlgYtsyB5ppuOHdfWzU2B8cYATwTz54c=
github.com/projectdiscovery/wappalyzergo v0.0.122/go.mod h1:qW0PP+UBMcdQBBnwk+X6YYFs6huKNvn2BOVs4vQPru0=
github.com/projectdiscovery/yamldoc-go v1.0.4 h1:eZoESapnMw6WAHiVgRwNqvbJEfNHEH148uthhFbG5jE=
diff --git a/internal/pdcp/writer.go b/internal/pdcp/writer.go
index a76d4ea31a..6e05410355 100644
--- a/internal/pdcp/writer.go
+++ b/internal/pdcp/writer.go
@@ -19,6 +19,7 @@ import (
"github.com/projectdiscovery/retryablehttp-go"
pdcpauth "github.com/projectdiscovery/utils/auth/pdcp"
errorutil "github.com/projectdiscovery/utils/errors"
+ unitutils "github.com/projectdiscovery/utils/unit"
updateutils "github.com/projectdiscovery/utils/update"
urlutil "github.com/projectdiscovery/utils/url"
)
@@ -26,8 +27,8 @@ import (
const (
uploadEndpoint = "/v1/scans/import"
appendEndpoint = "/v1/scans/%s/import"
- flushTimer = time.Duration(1) * time.Minute
- MaxChunkSize = 1024 * 1024 * 4 // 4 MB
+ flushTimer = time.Minute
+ MaxChunkSize = 4 * unitutils.Mega // 4 MB
xidRe = `^[a-z0-9]{20}$`
)
diff --git a/pkg/output/output.go b/pkg/output/output.go
index 044d164f1f..23b2f0ec91 100644
--- a/pkg/output/output.go
+++ b/pkg/output/output.go
@@ -30,6 +30,7 @@ import (
"github.com/projectdiscovery/nuclei/v3/pkg/utils"
fileutil "github.com/projectdiscovery/utils/file"
osutils "github.com/projectdiscovery/utils/os"
+ unitutils "github.com/projectdiscovery/utils/unit"
)
// Writer is an interface which writes output to somewhere for nuclei events.
@@ -411,7 +412,7 @@ func (w *StandardWriter) WriteFailure(wrappedEvent *InternalWrappedEvent) error
return w.Write(data)
}
-var maxTemplateFileSizeForEncoding = 1024 * 1024
+var maxTemplateFileSizeForEncoding = unitutils.Mega
func (w *StandardWriter) encodeTemplate(templatePath string) string {
data, err := os.ReadFile(templatePath)
diff --git a/pkg/protocols/common/automaticscan/automaticscan.go b/pkg/protocols/common/automaticscan/automaticscan.go
index 2a7988cb04..a5e51c177a 100644
--- a/pkg/protocols/common/automaticscan/automaticscan.go
+++ b/pkg/protocols/common/automaticscan/automaticscan.go
@@ -32,13 +32,14 @@ import (
sliceutil "github.com/projectdiscovery/utils/slice"
stringsutil "github.com/projectdiscovery/utils/strings"
syncutil "github.com/projectdiscovery/utils/sync"
+ unitutils "github.com/projectdiscovery/utils/unit"
wappalyzer "github.com/projectdiscovery/wappalyzergo"
"gopkg.in/yaml.v2"
)
const (
mappingFilename = "wappalyzer-mapping.yml"
- maxDefaultBody = 4 * 1024 * 1024 // 4MB
+ maxDefaultBody = 4 * unitutils.Mega
)
// Options contains configuration options for automatic scan service
diff --git a/pkg/protocols/http/request.go b/pkg/protocols/http/request.go
index df914a785c..fc3df039ac 100644
--- a/pkg/protocols/http/request.go
+++ b/pkg/protocols/http/request.go
@@ -42,6 +42,7 @@ import (
"github.com/projectdiscovery/utils/reader"
sliceutil "github.com/projectdiscovery/utils/slice"
stringsutil "github.com/projectdiscovery/utils/strings"
+ unitutils "github.com/projectdiscovery/utils/unit"
urlutil "github.com/projectdiscovery/utils/url"
)
@@ -53,7 +54,7 @@ const (
)
var (
- MaxBodyRead = int64(10 * 1024 * 1024) // 10MB
+ MaxBodyRead = 10 * unitutils.Mega
// ErrMissingVars is error occured when variables are missing
ErrMissingVars = errors.New("stop execution due to unresolved variables")
)
@@ -557,7 +558,7 @@ func (request *Request) ExecuteWithResults(input *contextargs.Context, dynamicVa
return requestErr
}
-const drainReqSize = int64(8 * 1024)
+const drainReqSize = int64(8 * unitutils.Kilo)
// executeRequest executes the actual generated request and returns error if occurred
func (request *Request) executeRequest(input *contextargs.Context, generatedRequest *generatedRequest, previousEvent output.InternalEvent, hasInteractMatchers bool, processEvent protocols.OutputEventCallback, requestCount int) (err error) {
@@ -809,15 +810,15 @@ func (request *Request) executeRequest(input *contextargs.Context, generatedRequ
// define max body read limit
maxBodylimit := MaxBodyRead // 10MB
if request.MaxSize > 0 {
- maxBodylimit = int64(request.MaxSize)
+ maxBodylimit = request.MaxSize
}
if request.options.Options.ResponseReadSize != 0 {
- maxBodylimit = int64(request.options.Options.ResponseReadSize)
+ maxBodylimit = request.options.Options.ResponseReadSize
}
// respChain is http response chain that reads response body
// efficiently by reusing buffers and does all decoding and optimizations
- respChain := httpUtils.NewResponseChain(resp, maxBodylimit)
+ respChain := httpUtils.NewResponseChain(resp, int64(maxBodylimit))
defer respChain.Close() // reuse buffers
// we only intend to log/save the final redirected response
diff --git a/pkg/protocols/offlinehttp/request.go b/pkg/protocols/offlinehttp/request.go
index 4a440c167f..e913e02d8f 100644
--- a/pkg/protocols/offlinehttp/request.go
+++ b/pkg/protocols/offlinehttp/request.go
@@ -17,11 +17,12 @@ import (
templateTypes "github.com/projectdiscovery/nuclei/v3/pkg/templates/types"
"github.com/projectdiscovery/utils/conversion"
syncutil "github.com/projectdiscovery/utils/sync"
+ unitutils "github.com/projectdiscovery/utils/unit"
)
var _ protocols.Request = &Request{}
-const maxSize = 5 * 1024 * 1024
+const maxSize = 5 * unitutils.Mega
// Type returns the type of the protocol request
func (request *Request) Type() templateTypes.ProtocolType {
diff --git a/pkg/protocols/protocols.go b/pkg/protocols/protocols.go
index 6b4904c8d8..7804ea3c29 100644
--- a/pkg/protocols/protocols.go
+++ b/pkg/protocols/protocols.go
@@ -33,10 +33,11 @@ import (
"github.com/projectdiscovery/nuclei/v3/pkg/scan"
templateTypes "github.com/projectdiscovery/nuclei/v3/pkg/templates/types"
"github.com/projectdiscovery/nuclei/v3/pkg/types"
+ unitutils "github.com/projectdiscovery/utils/unit"
)
var (
- MaxTemplateFileSizeForEncoding = 1024 * 1024
+ MaxTemplateFileSizeForEncoding = unitutils.Mega
)
// Executer is an interface implemented any protocol based request executer.
diff --git a/pkg/reporting/format/format_utils.go b/pkg/reporting/format/format_utils.go
index 62a3d75fe8..d5ec0adfb1 100644
--- a/pkg/reporting/format/format_utils.go
+++ b/pkg/reporting/format/format_utils.go
@@ -12,6 +12,7 @@ import (
"github.com/projectdiscovery/nuclei/v3/pkg/reporting/exporters/markdown/util"
"github.com/projectdiscovery/nuclei/v3/pkg/types"
"github.com/projectdiscovery/nuclei/v3/pkg/utils"
+ unitutils "github.com/projectdiscovery/utils/unit"
)
// Summary returns a formatted built one line summary of the event
@@ -71,7 +72,7 @@ func CreateReportDescription(event *output.ResultEvent, formatter ResultFormatte
if event.Response != "" {
var responseString string
// If the response is larger than 5 kb, truncate it before writing.
- maxKbSize := 5 * 1024
+ maxKbSize := 5 * unitutils.Kilo
if len(event.Response) > maxKbSize {
responseString = event.Response[:maxKbSize]
responseString += ".... Truncated ...."
diff --git a/pkg/testutils/testutils.go b/pkg/testutils/testutils.go
index 68410f7f1d..a54fbbcf40 100644
--- a/pkg/testutils/testutils.go
+++ b/pkg/testutils/testutils.go
@@ -23,6 +23,7 @@ import (
protocolUtils "github.com/projectdiscovery/nuclei/v3/pkg/protocols/utils"
"github.com/projectdiscovery/nuclei/v3/pkg/types"
"github.com/projectdiscovery/nuclei/v3/pkg/utils"
+ unitutils "github.com/projectdiscovery/utils/unit"
)
// Init initializes the protocols and their configurations
@@ -201,7 +202,7 @@ func (m *MockOutputWriter) WriteFailure(wrappedEvent *output.InternalWrappedEven
return m.Write(data)
}
-var maxTemplateFileSizeForEncoding = 1024 * 1024
+var maxTemplateFileSizeForEncoding = unitutils.Mega
func (w *MockOutputWriter) encodeTemplate(templatePath string) string {
data, err := os.ReadFile(templatePath)
diff --git a/pkg/types/types.go b/pkg/types/types.go
index 0c31769ee1..577461b217 100644
--- a/pkg/types/types.go
+++ b/pkg/types/types.go
@@ -15,6 +15,7 @@ import (
errorutil "github.com/projectdiscovery/utils/errors"
fileutil "github.com/projectdiscovery/utils/file"
folderutil "github.com/projectdiscovery/utils/folder"
+ unitutils "github.com/projectdiscovery/utils/unit"
)
var (
@@ -435,8 +436,8 @@ func DefaultOptions() *Options {
Timeout: 5,
Retries: 1,
MaxHostError: 30,
- ResponseReadSize: 10 * 1024 * 1024,
- ResponseSaveSize: 1024 * 1024,
+ ResponseReadSize: 10 * unitutils.Mega,
+ ResponseSaveSize: unitutils.Mega,
ResponseReadTimeout: 5 * time.Second,
}
}
diff --git a/pkg/utils/monitor/monitor.go b/pkg/utils/monitor/monitor.go
index 6441fdf20d..cefb9506b8 100644
--- a/pkg/utils/monitor/monitor.go
+++ b/pkg/utils/monitor/monitor.go
@@ -16,6 +16,7 @@ import (
"github.com/DataDog/gostackparse"
"github.com/projectdiscovery/gologger"
permissionutil "github.com/projectdiscovery/utils/permission"
+ unitutils "github.com/projectdiscovery/utils/unit"
"github.com/rs/xid"
)
@@ -118,7 +119,7 @@ func (s *Agent) monitorWorker(cancel context.CancelFunc) {
// getStack returns full stack trace of the program
var getStack = func(all bool) []byte {
- for i := 1024 * 1024; ; i *= 2 {
+ for i := unitutils.Mega; ; i *= 2 {
buf := make([]byte, i)
if n := runtime.Stack(buf, all); n < i {
return buf[:n-1]
From 47ca8fe8427a5cd08886c08cf84cc2feebe3d1cd Mon Sep 17 00:00:00 2001
From: Mzack9999
Date: Wed, 15 May 2024 21:25:40 +0200
Subject: [PATCH 006/283] fix non gc-able dialer
closes #5165
---
go.mod | 2 +-
go.sum | 4 ++--
lib/sdk.go | 6 +++---
lib/sdk_private.go | 6 +++++-
pkg/protocols/common/protocolinit/init.go | 3 +--
pkg/protocols/common/protocolstate/state.go | 5 +++++
.../http/httpclientpool/clientpool.go | 20 ++++++-------------
pkg/protocols/http/request.go | 4 ++--
8 files changed, 25 insertions(+), 25 deletions(-)
diff --git a/go.mod b/go.mod
index 39156d2f6d..98973239e5 100644
--- a/go.mod
+++ b/go.mod
@@ -99,7 +99,7 @@ require (
github.com/redis/go-redis/v9 v9.1.0
github.com/seh-msft/burpxml v1.0.1
github.com/stretchr/testify v1.9.0
- github.com/tarunKoyalwar/goleak v0.0.0-20240426214851-746d64600adc
+ github.com/tarunKoyalwar/goleak v0.0.0-20240429141123-0efa90dbdcf9
github.com/zmap/zgrab2 v0.1.8-0.20230806160807-97ba87c0e706
golang.org/x/term v0.19.0
gopkg.in/yaml.v3 v3.0.1
diff --git a/go.sum b/go.sum
index 4a95dbd040..281ee725da 100644
--- a/go.sum
+++ b/go.sum
@@ -1016,8 +1016,8 @@ github.com/stretchr/testify v1.9.0 h1:HtqpIVDClZ4nwg75+f6Lvsy/wHu+3BoSGCbBAcpTsT
github.com/stretchr/testify v1.9.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY=
github.com/syndtr/goleveldb v1.0.0 h1:fBdIW9lB4Iz0n9khmH8w27SJ3QEJ7+IgjPEwGSZiFdE=
github.com/syndtr/goleveldb v1.0.0/go.mod h1:ZVVdQEZoIme9iO1Ch2Jdy24qqXrMMOU6lpPAyBWyWuQ=
-github.com/tarunKoyalwar/goleak v0.0.0-20240426214851-746d64600adc h1:/5P5I7oDqdLee8W9Moof0xSD8tT1qEVzhObSI9CqHkg=
-github.com/tarunKoyalwar/goleak v0.0.0-20240426214851-746d64600adc/go.mod h1:uQdBQGrE1fZ2EyOs0pLcCDd1bBV4rSThieuIIGhXZ50=
+github.com/tarunKoyalwar/goleak v0.0.0-20240429141123-0efa90dbdcf9 h1:GXIyLuIJ5Qk46lI8WJ83qHBZKUI3zhmMmuoY9HICUIQ=
+github.com/tarunKoyalwar/goleak v0.0.0-20240429141123-0efa90dbdcf9/go.mod h1:uQdBQGrE1fZ2EyOs0pLcCDd1bBV4rSThieuIIGhXZ50=
github.com/tidwall/assert v0.1.0 h1:aWcKyRBUAdLoVebxo95N7+YZVTFF/ASTr7BN4sLP6XI=
github.com/tidwall/assert v0.1.0/go.mod h1:QLYtGyeqse53vuELQheYl9dngGCJQ+mTtlxcktb+Kj8=
github.com/tidwall/btree v1.7.0 h1:L1fkJH/AuEh5zBnnBbmTwQ5Lt+bRJ5A8EWecslvo9iI=
diff --git a/lib/sdk.go b/lib/sdk.go
index 63925f47ca..7a28cbc53e 100644
--- a/lib/sdk.go
+++ b/lib/sdk.go
@@ -18,7 +18,7 @@ import (
"github.com/projectdiscovery/nuclei/v3/pkg/protocols"
"github.com/projectdiscovery/nuclei/v3/pkg/protocols/common/hosterrorscache"
"github.com/projectdiscovery/nuclei/v3/pkg/protocols/common/interactsh"
- "github.com/projectdiscovery/nuclei/v3/pkg/protocols/common/protocolstate"
+ "github.com/projectdiscovery/nuclei/v3/pkg/protocols/common/protocolinit"
"github.com/projectdiscovery/nuclei/v3/pkg/protocols/headless/engine"
"github.com/projectdiscovery/nuclei/v3/pkg/reporting"
"github.com/projectdiscovery/nuclei/v3/pkg/templates"
@@ -206,8 +206,6 @@ func (e *NucleiEngine) Close() {
if e.rateLimiter != nil {
e.rateLimiter.Stop()
}
- // close global shared resources
- protocolstate.Close()
if e.inputProvider != nil {
e.inputProvider.Close()
}
@@ -217,6 +215,8 @@ func (e *NucleiEngine) Close() {
if e.httpxClient != nil {
_ = e.httpxClient.Close()
}
+ // close global shared resources
+ protocolinit.Close()
}
// ExecuteCallbackWithCtx executes templates on targets and calls callback on each result(only if results are found)
diff --git a/lib/sdk_private.go b/lib/sdk_private.go
index 13e8746510..6f8c929e38 100644
--- a/lib/sdk_private.go
+++ b/lib/sdk_private.go
@@ -35,7 +35,7 @@ import (
"github.com/projectdiscovery/ratelimit"
)
-var sharedInit sync.Once = sync.Once{}
+var sharedInit *sync.Once
// applyRequiredDefaults to options
func (e *NucleiEngine) applyRequiredDefaults() {
@@ -118,6 +118,10 @@ func (e *NucleiEngine) init() error {
e.parser = templates.NewParser()
+ if sharedInit == nil || protocolstate.ShouldInit() {
+ sharedInit = &sync.Once{}
+ }
+
sharedInit.Do(func() {
_ = protocolstate.Init(e.opts)
_ = protocolinit.Init(e.opts)
diff --git a/pkg/protocols/common/protocolinit/init.go b/pkg/protocols/common/protocolinit/init.go
index 2ab5d7ca9c..c8268337f5 100644
--- a/pkg/protocols/common/protocolinit/init.go
+++ b/pkg/protocols/common/protocolinit/init.go
@@ -13,7 +13,6 @@ import (
// Init initializes the client pools for the protocols
func Init(options *types.Options) error {
-
if err := protocolstate.Init(options); err != nil {
return err
}
@@ -39,5 +38,5 @@ func Init(options *types.Options) error {
}
func Close() {
- protocolstate.Dialer.Close()
+ protocolstate.Close()
}
diff --git a/pkg/protocols/common/protocolstate/state.go b/pkg/protocols/common/protocolstate/state.go
index 67820ec074..02e30e06fd 100644
--- a/pkg/protocols/common/protocolstate/state.go
+++ b/pkg/protocols/common/protocolstate/state.go
@@ -22,6 +22,10 @@ var (
Dialer *fastdialer.Dialer
)
+func ShouldInit() bool {
+ return Dialer == nil
+}
+
// Init creates the Dialer instance based on user configuration
func Init(options *types.Options) error {
if Dialer != nil {
@@ -210,5 +214,6 @@ func Close() {
if Dialer != nil {
Dialer.Close()
}
+ Dialer = nil
StopActiveMemGuardian()
}
diff --git a/pkg/protocols/http/httpclientpool/clientpool.go b/pkg/protocols/http/httpclientpool/clientpool.go
index 3e2baf55a5..96b0939511 100644
--- a/pkg/protocols/http/httpclientpool/clientpool.go
+++ b/pkg/protocols/http/httpclientpool/clientpool.go
@@ -16,7 +16,6 @@ import (
"golang.org/x/net/proxy"
"golang.org/x/net/publicsuffix"
- "github.com/projectdiscovery/fastdialer/fastdialer"
"github.com/projectdiscovery/fastdialer/fastdialer/ja3/impersonate"
"github.com/projectdiscovery/nuclei/v3/pkg/protocols/common/protocolstate"
"github.com/projectdiscovery/nuclei/v3/pkg/protocols/utils"
@@ -28,9 +27,6 @@ import (
)
var (
- // Dialer is a copy of the fastdialer from protocolstate
- Dialer *fastdialer.Dialer
-
rawHttpClient *rawhttp.Client
forceMaxRedirects int
normalClient *retryablehttp.Client
@@ -146,8 +142,8 @@ func GetRawHTTP(options *types.Options) *rawhttp.Client {
rawHttpOptions.Proxy = types.ProxyURL
} else if types.ProxySocksURL != "" {
rawHttpOptions.Proxy = types.ProxySocksURL
- } else if Dialer != nil {
- rawHttpOptions.FastDialer = Dialer
+ } else if protocolstate.Dialer != nil {
+ rawHttpOptions.FastDialer = protocolstate.Dialer
}
rawHttpOptions.Timeout = GetHttpTimeout(options)
rawHttpClient = rawhttp.NewClient(rawHttpOptions)
@@ -167,10 +163,6 @@ func Get(options *types.Options, configuration *Configuration) (*retryablehttp.C
func wrappedGet(options *types.Options, configuration *Configuration) (*retryablehttp.Client, error) {
var err error
- if Dialer == nil {
- Dialer = protocolstate.Dialer
- }
-
hash := configuration.Hash()
if client, ok := clientPool.Get(hash); ok {
return client, nil
@@ -237,15 +229,15 @@ func wrappedGet(options *types.Options, configuration *Configuration) (*retryabl
transport := &http.Transport{
ForceAttemptHTTP2: options.ForceAttemptHTTP2,
- DialContext: Dialer.Dial,
+ DialContext: protocolstate.Dialer.Dial,
DialTLSContext: func(ctx context.Context, network, addr string) (net.Conn, error) {
if options.TlsImpersonate {
- return Dialer.DialTLSWithConfigImpersonate(ctx, network, addr, tlsConfig, impersonate.Random, nil)
+ return protocolstate.Dialer.DialTLSWithConfigImpersonate(ctx, network, addr, tlsConfig, impersonate.Random, nil)
}
if options.HasClientCertificates() || options.ForceAttemptHTTP2 {
- return Dialer.DialTLSWithConfig(ctx, network, addr, tlsConfig)
+ return protocolstate.Dialer.DialTLSWithConfig(ctx, network, addr, tlsConfig)
}
- return Dialer.DialTLS(ctx, network, addr)
+ return protocolstate.Dialer.DialTLS(ctx, network, addr)
},
MaxIdleConns: maxIdleConns,
MaxIdleConnsPerHost: maxIdleConnsPerHost,
diff --git a/pkg/protocols/http/request.go b/pkg/protocols/http/request.go
index fc3df039ac..c46db8f4b5 100644
--- a/pkg/protocols/http/request.go
+++ b/pkg/protocols/http/request.go
@@ -777,7 +777,7 @@ func (request *Request) executeRequest(input *contextargs.Context, generatedRequ
if input.MetaInput.CustomIP != "" {
outputEvent["ip"] = input.MetaInput.CustomIP
} else {
- outputEvent["ip"] = httpclientpool.Dialer.GetDialedIP(hostname)
+ outputEvent["ip"] = protocolstate.Dialer.GetDialedIP(hostname)
}
if len(generatedRequest.interactshURLs) > 0 {
@@ -873,7 +873,7 @@ func (request *Request) executeRequest(input *contextargs.Context, generatedRequ
if input.MetaInput.CustomIP != "" {
outputEvent["ip"] = input.MetaInput.CustomIP
} else {
- outputEvent["ip"] = httpclientpool.Dialer.GetDialedIP(hostname)
+ outputEvent["ip"] = protocolstate.Dialer.GetDialedIP(hostname)
}
if request.options.Interactsh != nil {
request.options.Interactsh.MakePlaceholders(generatedRequest.interactshURLs, outputEvent)
From b820fb946457516559937d42d21fa3419381e367 Mon Sep 17 00:00:00 2001
From: Mzack9999
Date: Wed, 15 May 2024 21:43:43 +0200
Subject: [PATCH 007/283] fixing shared cleanup
---
lib/multi.go | 2 +-
lib/sdk.go | 9 ++++++---
2 files changed, 7 insertions(+), 4 deletions(-)
diff --git a/lib/multi.go b/lib/multi.go
index 5b2c7d6776..2ede35aecf 100644
--- a/lib/multi.go
+++ b/lib/multi.go
@@ -123,7 +123,7 @@ func (e *ThreadSafeNucleiEngine) ExecuteNucleiWithOptsCtx(ctx context.Context, t
return err
}
}
- defer tmpEngine.Close()
+ defer tmpEngine.closeInternal()
// create ephemeral nuclei objects/instances/types using base nuclei engine
unsafeOpts, err := createEphemeralObjects(e.eng, tmpEngine.opts)
if err != nil {
diff --git a/lib/sdk.go b/lib/sdk.go
index 7a28cbc53e..4cfebb5c70 100644
--- a/lib/sdk.go
+++ b/lib/sdk.go
@@ -183,8 +183,7 @@ func (e *NucleiEngine) SignTemplate(tmplSigner *signer.TemplateSigner, data []by
return buff.Bytes(), err
}
-// Close all resources used by nuclei engine
-func (e *NucleiEngine) Close() {
+func (e *NucleiEngine) closeInternal() {
if e.interactshClient != nil {
e.interactshClient.Close()
}
@@ -215,7 +214,11 @@ func (e *NucleiEngine) Close() {
if e.httpxClient != nil {
_ = e.httpxClient.Close()
}
- // close global shared resources
+}
+
+// Close all resources used by nuclei engine
+func (e *NucleiEngine) Close() {
+ e.closeInternal()
protocolinit.Close()
}
From aaf5d262b2aefd07458edad4fcf316ac7c0260ad Mon Sep 17 00:00:00 2001
From: Mzack9999
Date: Thu, 16 May 2024 11:32:27 +0200
Subject: [PATCH 008/283] parallelize template loading
---
pkg/catalog/loader/loader.go | 126 +++++++++++++++++++----------------
1 file changed, 68 insertions(+), 58 deletions(-)
diff --git a/pkg/catalog/loader/loader.go b/pkg/catalog/loader/loader.go
index c719f71a5a..680a98c714 100644
--- a/pkg/catalog/loader/loader.go
+++ b/pkg/catalog/loader/loader.go
@@ -7,6 +7,7 @@ import (
"os"
"sort"
"strings"
+ "sync"
"github.com/logrusorgru/aurora"
"github.com/pkg/errors"
@@ -441,75 +442,84 @@ func (store *Store) LoadTemplatesWithTags(templatesList, tags []string) []*templ
}
}
+ var wgLoadTemplates sync.WaitGroup
+
for templatePath := range templatePathMap {
- loaded, err := store.config.ExecutorOptions.Parser.LoadTemplate(templatePath, store.tagFilter, tags, store.config.Catalog)
- if loaded || store.pathFilter.MatchIncluded(templatePath) {
- parsed, err := templates.Parse(templatePath, store.preprocessor, store.config.ExecutorOptions)
- if err != nil {
- // exclude templates not compatible with offline matching from total runtime warning stats
- if !errors.Is(err, templates.ErrIncompatibleWithOfflineMatching) {
- stats.Increment(templates.RuntimeWarningsStats)
- }
- gologger.Warning().Msgf("Could not parse template %s: %s\n", templatePath, err)
- } else if parsed != nil {
- if !parsed.Verified && store.config.ExecutorOptions.Options.DisableUnsignedTemplates {
- // skip unverified templates when prompted to
- stats.Increment(templates.SkippedUnsignedStats)
- continue
- }
- // if template has request signature like aws then only signed and verified templates are allowed
- if parsed.UsesRequestSignature() && !parsed.Verified {
- stats.Increment(templates.SkippedRequestSignatureStats)
- continue
- }
- // DAST only templates
- if store.config.ExecutorOptions.Options.DAST {
- // check if the template is a DAST template
- if parsed.IsFuzzing() {
- loadTemplate(parsed)
+ wgLoadTemplates.Add(1)
+ go func(templatePath string) {
+ defer wgLoadTemplates.Done()
+
+ loaded, err := store.config.ExecutorOptions.Parser.LoadTemplate(templatePath, store.tagFilter, tags, store.config.Catalog)
+ if loaded || store.pathFilter.MatchIncluded(templatePath) {
+ parsed, err := templates.Parse(templatePath, store.preprocessor, store.config.ExecutorOptions)
+ if err != nil {
+ // exclude templates not compatible with offline matching from total runtime warning stats
+ if !errors.Is(err, templates.ErrIncompatibleWithOfflineMatching) {
+ stats.Increment(templates.RuntimeWarningsStats)
}
- } else if len(parsed.RequestsHeadless) > 0 && !store.config.ExecutorOptions.Options.Headless {
- // donot include headless template in final list if headless flag is not set
- stats.Increment(templates.ExcludedHeadlessTmplStats)
- if config.DefaultConfig.LogAllEvents {
- gologger.Print().Msgf("[%v] Headless flag is required for headless template '%s'.\n", aurora.Yellow("WRN").String(), templatePath)
+ gologger.Warning().Msgf("Could not parse template %s: %s\n", templatePath, err)
+ } else if parsed != nil {
+ if !parsed.Verified && store.config.ExecutorOptions.Options.DisableUnsignedTemplates {
+ // skip unverified templates when prompted to
+ stats.Increment(templates.SkippedUnsignedStats)
+ return
}
- } else if len(parsed.RequestsCode) > 0 && !store.config.ExecutorOptions.Options.EnableCodeTemplates {
- // donot include 'Code' protocol custom template in final list if code flag is not set
- stats.Increment(templates.ExcludedCodeTmplStats)
- if config.DefaultConfig.LogAllEvents {
- gologger.Print().Msgf("[%v] Code flag is required for code protocol template '%s'.\n", aurora.Yellow("WRN").String(), templatePath)
+ // if template has request signature like aws then only signed and verified templates are allowed
+ if parsed.UsesRequestSignature() && !parsed.Verified {
+ stats.Increment(templates.SkippedRequestSignatureStats)
+ return
}
- } else if len(parsed.RequestsCode) > 0 && !parsed.Verified && len(parsed.Workflows) == 0 {
- // donot include unverified 'Code' protocol custom template in final list
- stats.Increment(templates.SkippedCodeTmplTamperedStats)
- // these will be skipped so increment skip counter
- stats.Increment(templates.SkippedUnsignedStats)
- if config.DefaultConfig.LogAllEvents {
- gologger.Print().Msgf("[%v] Tampered/Unsigned template at %v.\n", aurora.Yellow("WRN").String(), templatePath)
- }
- } else if parsed.IsFuzzing() && !store.config.ExecutorOptions.Options.DAST {
- stats.Increment(templates.ExludedDastTmplStats)
- if config.DefaultConfig.LogAllEvents {
- gologger.Print().Msgf("[%v] -dast flag is required for DAST template '%s'.\n", aurora.Yellow("WRN").String(), templatePath)
+ // DAST only templates
+ if store.config.ExecutorOptions.Options.DAST {
+ // check if the template is a DAST template
+ if parsed.IsFuzzing() {
+ loadTemplate(parsed)
+ }
+ } else if len(parsed.RequestsHeadless) > 0 && !store.config.ExecutorOptions.Options.Headless {
+ // donot include headless template in final list if headless flag is not set
+ stats.Increment(templates.ExcludedHeadlessTmplStats)
+ if config.DefaultConfig.LogAllEvents {
+ gologger.Print().Msgf("[%v] Headless flag is required for headless template '%s'.\n", aurora.Yellow("WRN").String(), templatePath)
+ }
+ } else if len(parsed.RequestsCode) > 0 && !store.config.ExecutorOptions.Options.EnableCodeTemplates {
+ // donot include 'Code' protocol custom template in final list if code flag is not set
+ stats.Increment(templates.ExcludedCodeTmplStats)
+ if config.DefaultConfig.LogAllEvents {
+ gologger.Print().Msgf("[%v] Code flag is required for code protocol template '%s'.\n", aurora.Yellow("WRN").String(), templatePath)
+ }
+ } else if len(parsed.RequestsCode) > 0 && !parsed.Verified && len(parsed.Workflows) == 0 {
+ // donot include unverified 'Code' protocol custom template in final list
+ stats.Increment(templates.SkippedCodeTmplTamperedStats)
+ // these will be skipped so increment skip counter
+ stats.Increment(templates.SkippedUnsignedStats)
+ if config.DefaultConfig.LogAllEvents {
+ gologger.Print().Msgf("[%v] Tampered/Unsigned template at %v.\n", aurora.Yellow("WRN").String(), templatePath)
+ }
+ } else if parsed.IsFuzzing() && !store.config.ExecutorOptions.Options.DAST {
+ stats.Increment(templates.ExludedDastTmplStats)
+ if config.DefaultConfig.LogAllEvents {
+ gologger.Print().Msgf("[%v] -dast flag is required for DAST template '%s'.\n", aurora.Yellow("WRN").String(), templatePath)
+ }
+ } else {
+ loadTemplate(parsed)
}
- } else {
- loadTemplate(parsed)
}
}
- }
- if err != nil {
- if strings.Contains(err.Error(), templates.ErrExcluded.Error()) {
- stats.Increment(templates.TemplatesExcludedStats)
- if config.DefaultConfig.LogAllEvents {
- gologger.Print().Msgf("[%v] %v\n", aurora.Yellow("WRN").String(), err.Error())
+ if err != nil {
+ if strings.Contains(err.Error(), templates.ErrExcluded.Error()) {
+ stats.Increment(templates.TemplatesExcludedStats)
+ if config.DefaultConfig.LogAllEvents {
+ gologger.Print().Msgf("[%v] %v\n", aurora.Yellow("WRN").String(), err.Error())
+ }
+ return
}
- continue
+ gologger.Warning().Msg(err.Error())
}
- gologger.Warning().Msg(err.Error())
- }
+ }(templatePath)
}
+ wgLoadTemplates.Wait()
+
sort.SliceStable(loadedTemplates, func(i, j int) bool {
return loadedTemplates[i].Path < loadedTemplates[j].Path
})
From 3d5605530bc210557f6988c34cc73a0c9832a463 Mon Sep 17 00:00:00 2001
From: mzack9999
Date: Thu, 16 May 2024 13:42:40 +0200
Subject: [PATCH 009/283] race
---
pkg/utils/stats/stats.go | 46 ++++++++++++++--------------------------
1 file changed, 16 insertions(+), 30 deletions(-)
diff --git a/pkg/utils/stats/stats.go b/pkg/utils/stats/stats.go
index 590608d1e2..b6f9e9b36d 100644
--- a/pkg/utils/stats/stats.go
+++ b/pkg/utils/stats/stats.go
@@ -2,23 +2,22 @@ package stats
import (
"fmt"
- "sync"
"sync/atomic"
"github.com/logrusorgru/aurora"
"github.com/projectdiscovery/gologger"
+ mapsutil "github.com/projectdiscovery/utils/maps"
)
// Storage is a storage for storing statistics information
// about the nuclei engine displaying it at user-defined intervals.
type Storage struct {
- data map[string]*storageDataItem
- mutex *sync.RWMutex
+ data *mapsutil.SyncLockMap[string, *storageDataItem]
}
type storageDataItem struct {
description string
- value int64
+ value atomic.Int64
}
var Default *Storage
@@ -59,38 +58,32 @@ func GetValue(name string) int64 {
// New creates a new storage object
func New() *Storage {
- return &Storage{data: make(map[string]*storageDataItem), mutex: &sync.RWMutex{}}
+ data := mapsutil.NewSyncLockMap[string, *storageDataItem]()
+ return &Storage{data: data}
}
// NewEntry creates a new entry in the storage object
func (s *Storage) NewEntry(name, description string) {
- s.mutex.Lock()
- s.data[name] = &storageDataItem{description: description, value: 0}
- s.mutex.Unlock()
+ s.data.Set(name, &storageDataItem{description: description, value: atomic.Int64{}})
}
// Increment increments the value for a name string
func (s *Storage) Increment(name string) {
- s.mutex.RLock()
- data, ok := s.data[name]
- s.mutex.RUnlock()
+ data, ok := s.data.Get(name)
if !ok {
return
}
-
- atomic.AddInt64(&data.value, 1)
+ data.value.Add(1)
}
// Display displays the stats for a name
func (s *Storage) Display(name string) {
- s.mutex.RLock()
- data, ok := s.data[name]
- s.mutex.RUnlock()
+ data, ok := s.data.Get(name)
if !ok {
return
}
- dataValue := atomic.LoadInt64(&data.value)
+ dataValue := data.value.Load()
if dataValue == 0 {
return // don't show for nil stats
}
@@ -98,14 +91,12 @@ func (s *Storage) Display(name string) {
}
func (s *Storage) DisplayAsWarning(name string) {
- s.mutex.RLock()
- data, ok := s.data[name]
- s.mutex.RUnlock()
+ data, ok := s.data.Get(name)
if !ok {
return
}
- dataValue := atomic.LoadInt64(&data.value)
+ dataValue := data.value.Load()
if dataValue == 0 {
return // don't show for nil stats
}
@@ -115,14 +106,12 @@ func (s *Storage) DisplayAsWarning(name string) {
// ForceDisplayWarning forces the display of a warning
// regardless of current verbosity level
func (s *Storage) ForceDisplayWarning(name string) {
- s.mutex.RLock()
- data, ok := s.data[name]
- s.mutex.RUnlock()
+ data, ok := s.data.Get(name)
if !ok {
return
}
- dataValue := atomic.LoadInt64(&data.value)
+ dataValue := data.value.Load()
if dataValue == 0 {
return // don't show for nil stats
}
@@ -131,13 +120,10 @@ func (s *Storage) ForceDisplayWarning(name string) {
// GetValue returns the value for a set variable
func (s *Storage) GetValue(name string) int64 {
- s.mutex.RLock()
- data, ok := s.data[name]
- s.mutex.RUnlock()
+ data, ok := s.data.Get(name)
if !ok {
return 0
}
- dataValue := atomic.LoadInt64(&data.value)
- return dataValue
+ return data.value.Load()
}
From 06b0219839da372d1d93fd3166491a125d219773 Mon Sep 17 00:00:00 2001
From: mzack9999
Date: Thu, 16 May 2024 13:48:28 +0200
Subject: [PATCH 010/283] lint
---
pkg/utils/stats/stats.go | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/pkg/utils/stats/stats.go b/pkg/utils/stats/stats.go
index b6f9e9b36d..f1edf2ea33 100644
--- a/pkg/utils/stats/stats.go
+++ b/pkg/utils/stats/stats.go
@@ -64,7 +64,7 @@ func New() *Storage {
// NewEntry creates a new entry in the storage object
func (s *Storage) NewEntry(name, description string) {
- s.data.Set(name, &storageDataItem{description: description, value: atomic.Int64{}})
+ _ = s.data.Set(name, &storageDataItem{description: description, value: atomic.Int64{}})
}
// Increment increments the value for a name string
From f1b739f2a5df1184c45e1e264f9d2818274cd0d9 Mon Sep 17 00:00:00 2001
From: mzack9999
Date: Thu, 16 May 2024 14:57:33 +0200
Subject: [PATCH 011/283] fixing race
---
go.mod | 2 +-
go.sum | 2 ++
pkg/catalog/loader/loader.go | 11 ++++++-----
3 files changed, 9 insertions(+), 6 deletions(-)
diff --git a/go.mod b/go.mod
index 98973239e5..3dfe4314bb 100644
--- a/go.mod
+++ b/go.mod
@@ -94,7 +94,7 @@ require (
github.com/projectdiscovery/tlsx v1.1.6
github.com/projectdiscovery/uncover v1.0.7
github.com/projectdiscovery/useragent v0.0.49
- github.com/projectdiscovery/utils v0.0.94-0.20240515132152-3908d4bb5d2c
+ github.com/projectdiscovery/utils v0.0.94-0.20240516124025-c3cf7603d51e
github.com/projectdiscovery/wappalyzergo v0.0.122
github.com/redis/go-redis/v9 v9.1.0
github.com/seh-msft/burpxml v1.0.1
diff --git a/go.sum b/go.sum
index 281ee725da..07ae0039f8 100644
--- a/go.sum
+++ b/go.sum
@@ -891,6 +891,8 @@ github.com/projectdiscovery/useragent v0.0.49 h1:wQc9i+Xy+mUMJ45Ralv1JsQImRWqEOE
github.com/projectdiscovery/useragent v0.0.49/go.mod h1:jQz6X/usiXrPYE6B/1uVKuzIrBJXgw9hLC9eeNy38+0=
github.com/projectdiscovery/utils v0.0.94-0.20240515132152-3908d4bb5d2c h1:vGsulLeJNcFjQTp+aSAO7auX5wriKrLrZ8+SILxtRcM=
github.com/projectdiscovery/utils v0.0.94-0.20240515132152-3908d4bb5d2c/go.mod h1:wxPi+kCsLm5JCLMkZJyGwS+4Mn4PaPHHf0ayE8JphOw=
+github.com/projectdiscovery/utils v0.0.94-0.20240516124025-c3cf7603d51e h1:PJbEUAFSSVT1egqY0J9V7yUi/ebz1agKBn+KPPUwBWA=
+github.com/projectdiscovery/utils v0.0.94-0.20240516124025-c3cf7603d51e/go.mod h1:wxPi+kCsLm5JCLMkZJyGwS+4Mn4PaPHHf0ayE8JphOw=
github.com/projectdiscovery/wappalyzergo v0.0.122 h1:xfNJ7VNzU/OGlgYtsyB5ppuOHdfWzU2B8cYATwTz54c=
github.com/projectdiscovery/wappalyzergo v0.0.122/go.mod h1:qW0PP+UBMcdQBBnwk+X6YYFs6huKNvn2BOVs4vQPru0=
github.com/projectdiscovery/yamldoc-go v1.0.4 h1:eZoESapnMw6WAHiVgRwNqvbJEfNHEH148uthhFbG5jE=
diff --git a/pkg/catalog/loader/loader.go b/pkg/catalog/loader/loader.go
index 680a98c714..e1d8371e35 100644
--- a/pkg/catalog/loader/loader.go
+++ b/pkg/catalog/loader/loader.go
@@ -24,6 +24,7 @@ import (
"github.com/projectdiscovery/nuclei/v3/pkg/workflows"
"github.com/projectdiscovery/retryablehttp-go"
errorutil "github.com/projectdiscovery/utils/errors"
+ sliceutil "github.com/projectdiscovery/utils/slice"
stringsutil "github.com/projectdiscovery/utils/strings"
urlutil "github.com/projectdiscovery/utils/url"
)
@@ -426,10 +427,10 @@ func (store *Store) LoadTemplatesWithTags(templatesList, tags []string) []*templ
store.logErroredTemplates(errs)
templatePathMap := store.pathFilter.Match(includedTemplates)
- loadedTemplates := make([]*templates.Template, 0, len(templatePathMap))
+ loadedTemplates := sliceutil.NewSyncSlice[*templates.Template]()
loadTemplate := func(tmpl *templates.Template) {
- loadedTemplates = append(loadedTemplates, tmpl)
+ loadedTemplates.Append(tmpl)
// increment signed/unsigned counters
if tmpl.Verified {
if tmpl.TemplateVerifier == "" {
@@ -520,11 +521,11 @@ func (store *Store) LoadTemplatesWithTags(templatesList, tags []string) []*templ
wgLoadTemplates.Wait()
- sort.SliceStable(loadedTemplates, func(i, j int) bool {
- return loadedTemplates[i].Path < loadedTemplates[j].Path
+ sort.SliceStable(loadedTemplates.Slice, func(i, j int) bool {
+ return loadedTemplates.Slice[i].Path < loadedTemplates.Slice[j].Path
})
- return loadedTemplates
+ return loadedTemplates.Slice
}
// IsHTTPBasedProtocolUsed returns true if http/headless protocol is being used for
From 4170e1cbb8653683977c3637b64c5203953bf9e4 Mon Sep 17 00:00:00 2001
From: Ice3man
Date: Mon, 20 May 2024 00:48:40 +0530
Subject: [PATCH 012/283] more goroutine leak fixes to nuclei (#5188)
* more goroutine leak fixes to nuclei
* run only dns templates for test
* updated httpx to dev
* dep update
---------
Co-authored-by: sandeep <8293321+ehsandeep@users.noreply.github.com>
---
go.mod | 28 +++++++--------
go.sum | 55 +++++++++++++++--------------
lib/tests/sdk_test.go | 2 +-
pkg/js/libs/postgres/postgres.go | 15 +++++---
pkg/protocols/http/httputils/spm.go | 2 ++
pkg/protocols/http/request.go | 24 +++++++++++--
pkg/utils/http_probe.go | 1 +
7 files changed, 77 insertions(+), 50 deletions(-)
diff --git a/go.mod b/go.mod
index 1002e41efd..36d655cd1b 100644
--- a/go.mod
+++ b/go.mod
@@ -21,9 +21,9 @@ require (
github.com/pkg/errors v0.9.1
github.com/projectdiscovery/clistats v0.0.20
github.com/projectdiscovery/fastdialer v0.0.71
- github.com/projectdiscovery/hmap v0.0.41
- github.com/projectdiscovery/interactsh v1.1.9
- github.com/projectdiscovery/rawhttp v0.1.47
+ github.com/projectdiscovery/hmap v0.0.42
+ github.com/projectdiscovery/interactsh v1.1.10-0.20240519152711-6a0cb98b1663
+ github.com/projectdiscovery/rawhttp v0.1.49
github.com/projectdiscovery/retryabledns v1.0.59
github.com/projectdiscovery/retryablehttp-go v1.0.60
github.com/projectdiscovery/yamldoc-go v1.0.4
@@ -38,9 +38,9 @@ require (
github.com/weppos/publicsuffix-go v0.30.2-0.20230730094716-a20f9abcc222
github.com/xanzy/go-gitlab v0.84.0
go.uber.org/multierr v1.11.0
- golang.org/x/net v0.24.0
+ golang.org/x/net v0.25.0
golang.org/x/oauth2 v0.11.0
- golang.org/x/text v0.14.0
+ golang.org/x/text v0.15.0
gopkg.in/yaml.v2 v2.4.0
)
@@ -78,30 +78,30 @@ require (
github.com/mholt/archiver v3.1.1+incompatible
github.com/ory/dockertest/v3 v3.10.0
github.com/praetorian-inc/fingerprintx v1.1.9
- github.com/projectdiscovery/dsl v0.0.52
+ github.com/projectdiscovery/dsl v0.0.56
github.com/projectdiscovery/fasttemplate v0.0.2
github.com/projectdiscovery/go-smb2 v0.0.0-20240129202741-052cc450c6cb
- github.com/projectdiscovery/goflags v0.1.50
+ github.com/projectdiscovery/goflags v0.1.52
github.com/projectdiscovery/gologger v1.1.12
github.com/projectdiscovery/gostruct v0.0.2
github.com/projectdiscovery/gozero v0.0.2
- github.com/projectdiscovery/httpx v1.6.0
+ github.com/projectdiscovery/httpx v1.6.1
github.com/projectdiscovery/mapcidr v1.1.34
github.com/projectdiscovery/n3iwf v0.0.0-20230523120440-b8cd232ff1f5
- github.com/projectdiscovery/ratelimit v0.0.39
+ github.com/projectdiscovery/ratelimit v0.0.40
github.com/projectdiscovery/rdap v0.9.1-0.20221108103045-9865884d1917
github.com/projectdiscovery/sarif v0.0.1
github.com/projectdiscovery/tlsx v1.1.6
github.com/projectdiscovery/uncover v1.0.7
github.com/projectdiscovery/useragent v0.0.49
- github.com/projectdiscovery/utils v0.0.92
- github.com/projectdiscovery/wappalyzergo v0.0.122
+ github.com/projectdiscovery/utils v0.0.93
+ github.com/projectdiscovery/wappalyzergo v0.1.0
github.com/redis/go-redis/v9 v9.1.0
github.com/seh-msft/burpxml v1.0.1
github.com/stretchr/testify v1.9.0
github.com/tarunKoyalwar/goleak v0.0.0-20240426214851-746d64600adc
github.com/zmap/zgrab2 v0.1.8-0.20230806160807-97ba87c0e706
- golang.org/x/term v0.19.0
+ golang.org/x/term v0.20.0
gopkg.in/yaml.v3 v3.0.1
moul.io/http2curl v1.0.0
)
@@ -303,10 +303,10 @@ require (
go.etcd.io/bbolt v1.3.8 // indirect
go.uber.org/zap v1.25.0 // indirect
goftp.io/server/v2 v2.0.1 // indirect
- golang.org/x/crypto v0.22.0 // indirect
+ golang.org/x/crypto v0.23.0 // indirect
golang.org/x/exp v0.0.0-20240119083558-1b970713d09a
golang.org/x/mod v0.14.0 // indirect
- golang.org/x/sys v0.19.0 // indirect
+ golang.org/x/sys v0.20.0 // indirect
golang.org/x/time v0.5.0 // indirect
golang.org/x/tools v0.17.0
google.golang.org/appengine v1.6.7 // indirect
diff --git a/go.sum b/go.sum
index dbc2f68aeb..2a8f67849f 100644
--- a/go.sum
+++ b/go.sum
@@ -835,8 +835,8 @@ github.com/projectdiscovery/cdncheck v1.0.9 h1:BS15gzj9gb5AVSKqTDzPamfSgStu7nJQO
github.com/projectdiscovery/cdncheck v1.0.9/go.mod h1:18SSl1w7rMj53CGeRIZTbDoa286a6xZIxGbaiEo4Fxs=
github.com/projectdiscovery/clistats v0.0.20 h1:5jO5SLiRJ7f0nDV0ndBNmBeesbROouPooH+DGMgoWq4=
github.com/projectdiscovery/clistats v0.0.20/go.mod h1:GJ2av0KnOvK0AISQnP8hyDclYIji1LVkx2l0pwnzAu4=
-github.com/projectdiscovery/dsl v0.0.52 h1:jvIvF+qN8+MbI1MHtWJJKfWqAZQlCExL3ob7SddQbZE=
-github.com/projectdiscovery/dsl v0.0.52/go.mod h1:xfcHwhy2HSaeGgh+1wqzOoCGm2XTdh5JzjBRBVHEMvI=
+github.com/projectdiscovery/dsl v0.0.56 h1:iVFIfDdGXkrXTh5sf6hRaxqTTEYiv/LnNjoOHJIfHiY=
+github.com/projectdiscovery/dsl v0.0.56/go.mod h1:3UBjPvtiy8c5E8oAUrKobMKjb3kEwQuGB5tqIN964Og=
github.com/projectdiscovery/fastdialer v0.0.71 h1:96j6Y65hDPZ9AzlYpp95hvIH5Yx/0OE2UTx+frWfnm4=
github.com/projectdiscovery/fastdialer v0.0.71/go.mod h1:b/oPPVSoLLD2N4W2/HrXbhQbyJVXqRw8CK1lenCUk64=
github.com/projectdiscovery/fasttemplate v0.0.2 h1:h2cISk5xDhlJEinlBQS6RRx0vOlOirB2y3Yu4PJzpiA=
@@ -845,20 +845,20 @@ github.com/projectdiscovery/freeport v0.0.5 h1:jnd3Oqsl4S8n0KuFkE5Hm8WGDP24ITBvm
github.com/projectdiscovery/freeport v0.0.5/go.mod h1:PY0bxSJ34HVy67LHIeF3uIutiCSDwOqKD8ruBkdiCwE=
github.com/projectdiscovery/go-smb2 v0.0.0-20240129202741-052cc450c6cb h1:rutG906Drtbpz4DwU5mhGIeOhRcktDH4cGQitGUMAsg=
github.com/projectdiscovery/go-smb2 v0.0.0-20240129202741-052cc450c6cb/go.mod h1:FLjF1DmZ+POoGEiIQdWuYVwS++C/GwpX8YaCsTSm1RY=
-github.com/projectdiscovery/goflags v0.1.50 h1:vhpM3mDTnXTktypUuVKWxgRpuBFJE+wzOr56rnOl4Bc=
-github.com/projectdiscovery/goflags v0.1.50/go.mod h1:3A8u7Q7e59S7f6QlJmMvKMXD/Tp+bfWre64gUpt6a6Q=
+github.com/projectdiscovery/goflags v0.1.52 h1:rVYZOtq7iA8e6ceyVZbp6OcuMhcwh5weiXSuDoXsivU=
+github.com/projectdiscovery/goflags v0.1.52/go.mod h1:tcBQ0EVGP4Wafza7gx57ZktkGxyfdLn+eQWUUQrV84c=
github.com/projectdiscovery/gologger v1.1.12 h1:uX/QkQdip4PubJjjG0+uk5DtyAi1ANPJUvpmimXqv4A=
github.com/projectdiscovery/gologger v1.1.12/go.mod h1:DI8nywPLERS5mo8QEA9E7gd5HZ3Je14SjJBH3F5/kLw=
github.com/projectdiscovery/gostruct v0.0.2 h1:s8gP8ApugGM4go1pA+sVlPDXaWqNP5BBDDSv7VEdG1M=
github.com/projectdiscovery/gostruct v0.0.2/go.mod h1:H86peL4HKwMXcQQtEa6lmC8FuD9XFt6gkNR0B/Mu5PE=
github.com/projectdiscovery/gozero v0.0.2 h1:8fJeaCjxL9tpm33uG/RsCQs6HGM/NE6eA3cjkilRQ+E=
github.com/projectdiscovery/gozero v0.0.2/go.mod h1:d8bZvDWW07LWNYWrwjZ4OO1I0cpkfqaysyDfSs9ibK8=
-github.com/projectdiscovery/hmap v0.0.41 h1:8IgTyDce3/2JzcfPVA4H+XpBRFfETULx8td3BMdSYVE=
-github.com/projectdiscovery/hmap v0.0.41/go.mod h1:bCrai6x5Eijqm2U+jtcH0wZX5ZcaZhcvzoMGTZgLAf0=
-github.com/projectdiscovery/httpx v1.6.0 h1:6g4UoSQpsOyZgaK+SMLLnZIAU0eYyTxBUwVl+jtm0JQ=
-github.com/projectdiscovery/httpx v1.6.0/go.mod h1:dzMzOWKfeofaXcXzac3O+YmuY24P0CRnviKGxvol6MM=
-github.com/projectdiscovery/interactsh v1.1.9 h1:b77SaSGrO+DtivmWwqGGY2dmNlQC3Zgmwlaj9L4Oqvc=
-github.com/projectdiscovery/interactsh v1.1.9/go.mod h1:0FRQXCildcTLq7Tsa4BVZAsFCXhpWs4C4quKWigXb5I=
+github.com/projectdiscovery/hmap v0.0.42 h1:+P8CC7gAeTG0phe0d1FB7i3Vl15v1K+dJApwX4rvMAM=
+github.com/projectdiscovery/hmap v0.0.42/go.mod h1:lbGBuL/bLoYWdlgphZmHXjZCYzteVDf4WfKsR/aH57c=
+github.com/projectdiscovery/httpx v1.6.1 h1:aqKzoSydzKAJBV3Sxbqt44nTah4fNS9GkSD4Dy6Fzlc=
+github.com/projectdiscovery/httpx v1.6.1/go.mod h1:RvibGQbm2Bh2DtuBDVXHLfauFDLtp7opKHyk65sz0ss=
+github.com/projectdiscovery/interactsh v1.1.10-0.20240519152711-6a0cb98b1663 h1:0ih5JJyTht2sv4J4lbfSPaUkKdFDci+Bwxg6vsyd2F4=
+github.com/projectdiscovery/interactsh v1.1.10-0.20240519152711-6a0cb98b1663/go.mod h1:gTrQCQfYlhoIM4X11+zwL7uRuraOUCoiE4Z6z1QD86g=
github.com/projectdiscovery/ldapserver v1.0.2-0.20240219154113-dcc758ebc0cb h1:MGtI4oE12ruWv11ZlPXXd7hl/uAaQZrFvrIDYDeVMd8=
github.com/projectdiscovery/ldapserver v1.0.2-0.20240219154113-dcc758ebc0cb/go.mod h1:vmgC0DTFCfoCLp0RAfsfYTZZan0QMVs+cmTbH6blfjk=
github.com/projectdiscovery/machineid v0.0.0-20240226150047-2e2c51e35983 h1:ZScLodGSezQVwsQDtBSMFp72WDq0nNN+KE/5DHKY5QE=
@@ -869,10 +869,10 @@ github.com/projectdiscovery/n3iwf v0.0.0-20230523120440-b8cd232ff1f5 h1:L/e8z8yw
github.com/projectdiscovery/n3iwf v0.0.0-20230523120440-b8cd232ff1f5/go.mod h1:pGW2ncnTxTxHtP9wzcIJAB+3/NMp6IiuQWd2NK7K+oc=
github.com/projectdiscovery/networkpolicy v0.0.8 h1:XvfBaBwSDNTesSfNQP9VLk3HX9I7x7gHm028TJ5XwI8=
github.com/projectdiscovery/networkpolicy v0.0.8/go.mod h1:xnjNqhemxUPxU+UD5Jgsc3+K8IVmcqT1SJeo6UzMtkI=
-github.com/projectdiscovery/ratelimit v0.0.39 h1:gMpkkuRSzsxNOxPOiD0aYTLQTOiqM/I0WZSZ7kmL+hc=
-github.com/projectdiscovery/ratelimit v0.0.39/go.mod h1:vR+q959CbyXCV/pD4bl4aGP2/R91BOKv1SyiHDOqeL0=
-github.com/projectdiscovery/rawhttp v0.1.47 h1:bjhZBK+7iuvcu0QTRJjdS4wP747+33Sd5gfgc4V9AWU=
-github.com/projectdiscovery/rawhttp v0.1.47/go.mod h1:2XA7xODWEGoHpEB8qzgqsV8yVL3cg5G63ZaT9ALwU4g=
+github.com/projectdiscovery/ratelimit v0.0.40 h1:SURkc7+ezUeGNt9PEQZOtd7nh2o2jKzmqcehrrf6DMU=
+github.com/projectdiscovery/ratelimit v0.0.40/go.mod h1:3Z1rK0bs4BSjdYNvIlbmZR1H8Z5MsXRz+QRCXGIB0HE=
+github.com/projectdiscovery/rawhttp v0.1.49 h1:OPP9R/UZx/GFrcPRUs9fOS1dLwhg+2o7p3dByzkIhWM=
+github.com/projectdiscovery/rawhttp v0.1.49/go.mod h1:aaAaMsdzHPfw4yU3nbeP7NI3vy1ZjgoXw7l+m4Tnt94=
github.com/projectdiscovery/rdap v0.9.1-0.20221108103045-9865884d1917 h1:m03X4gBVSorSzvmm0bFa7gDV4QNSOWPL/fgZ4kTXBxk=
github.com/projectdiscovery/rdap v0.9.1-0.20221108103045-9865884d1917/go.mod h1:JxXtZC9e195awe7EynrcnBJmFoad/BNDzW9mzFkK8Sg=
github.com/projectdiscovery/retryabledns v1.0.59 h1:8pMN+VibEBp29RIUior9LXUbx0RsBTjPC0008t2hfGU=
@@ -889,10 +889,10 @@ github.com/projectdiscovery/uncover v1.0.7 h1:ut+2lTuvmftmveqF5RTjMWAgyLj8ltPQC7
github.com/projectdiscovery/uncover v1.0.7/go.mod h1:HFXgm1sRPuoN0D4oATljPIdmbo/EEh1wVuxQqo/dwFE=
github.com/projectdiscovery/useragent v0.0.49 h1:wQc9i+Xy+mUMJ45Ralv1JsQImRWqEOEvpYUe6MchScg=
github.com/projectdiscovery/useragent v0.0.49/go.mod h1:jQz6X/usiXrPYE6B/1uVKuzIrBJXgw9hLC9eeNy38+0=
-github.com/projectdiscovery/utils v0.0.92 h1:lGCmjUJhzoNX4FQZWpp80058pRlD0/dYxLJOSs07EqY=
-github.com/projectdiscovery/utils v0.0.92/go.mod h1:d5uvD5qcRiK3qxZbBy9eatCqrCSuj9SObL04w/WgXSg=
-github.com/projectdiscovery/wappalyzergo v0.0.122 h1:xfNJ7VNzU/OGlgYtsyB5ppuOHdfWzU2B8cYATwTz54c=
-github.com/projectdiscovery/wappalyzergo v0.0.122/go.mod h1:qW0PP+UBMcdQBBnwk+X6YYFs6huKNvn2BOVs4vQPru0=
+github.com/projectdiscovery/utils v0.0.93 h1:IMZFsmQFYZUf7rxpBoZj+53FsNDC/vHsXA+4B4GuGeg=
+github.com/projectdiscovery/utils v0.0.93/go.mod h1:2+mWzk5FeYdK9imo5eLk6oVeih0G0wsTff1pzBAh9tk=
+github.com/projectdiscovery/wappalyzergo v0.1.0 h1:ZagOIKemBsNfCDRQeWavWEXtEjP8UiziuoRoPTwDAJ0=
+github.com/projectdiscovery/wappalyzergo v0.1.0/go.mod h1:wBYGKmA5BQp/NWsAy1q/jSH8N1LHWQ/LV26DuR+KzPM=
github.com/projectdiscovery/yamldoc-go v1.0.4 h1:eZoESapnMw6WAHiVgRwNqvbJEfNHEH148uthhFbG5jE=
github.com/projectdiscovery/yamldoc-go v1.0.4/go.mod h1:8PIPRcUD55UbtQdcfFR1hpIGRWG0P7alClXNGt1TBik=
github.com/prometheus/client_golang v0.9.1/go.mod h1:7SWBe2y4D6OKWSNQJUaRYU/AaXPKyh/dDVn+NZz0KFw=
@@ -1191,8 +1191,8 @@ golang.org/x/crypto v0.6.0/go.mod h1:OFC/31mSvZgRz0V1QTNCzfAI1aIRzbiufJtkMIlEp58
golang.org/x/crypto v0.7.0/go.mod h1:pYwdfH91IfpZVANVyUOhSIPZaFoJGxTFbZhFTx+dXZU=
golang.org/x/crypto v0.11.0/go.mod h1:xgJhtzW8F9jGdVFWZESrid1U1bjeNy4zgy5cRr/CIio=
golang.org/x/crypto v0.17.0/go.mod h1:gCAAfMLgwOJRpTjQ2zCCt2OcSfYMTeZVSRtQlPC7Nq4=
-golang.org/x/crypto v0.22.0 h1:g1v0xeRhjcugydODzvb3mEM9SQ0HGp9s/nh3COQ/C30=
-golang.org/x/crypto v0.22.0/go.mod h1:vr6Su+7cTlO45qkww3VDJlzDn0ctJvRgYbC2NvXHt+M=
+golang.org/x/crypto v0.23.0 h1:dIJU/v2J8Mdglj/8rJ6UUOM3Zc9zLZxVZwwxMooUSAI=
+golang.org/x/crypto v0.23.0/go.mod h1:CKFgDieR+mRhux2Lsu27y0fO304Db0wZe70UKqHu0v8=
golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA=
golang.org/x/exp v0.0.0-20190306152737-a1d7652674e8/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA=
golang.org/x/exp v0.0.0-20190510132918-efd6b22b2522/go.mod h1:ZjyILWgesfNpC6sMxTJOJm9Kp84zZh5NQWvqDGG3Qr8=
@@ -1285,8 +1285,8 @@ golang.org/x/net v0.8.0/go.mod h1:QVkue5JL9kW//ek3r6jTKnTFis1tRmNAW2P1shuFdJc=
golang.org/x/net v0.9.0/go.mod h1:d48xBJpPfHeWQsugry2m+kC02ZBRGRgulfHnEXEuWns=
golang.org/x/net v0.10.0/go.mod h1:0qNGK6F8kojg2nk9dLZ2mShWaEBan6FAoqfSigmmuDg=
golang.org/x/net v0.12.0/go.mod h1:zEVYFnQC7m/vmpQFELhcD1EWkZlX69l4oqgmer6hfKA=
-golang.org/x/net v0.24.0 h1:1PcaxkF854Fu3+lvBIx5SYn9wRlBzzcnHZSiaFFAb0w=
-golang.org/x/net v0.24.0/go.mod h1:2Q7sJY5mzlzWjKtYUEXSlBWCdyaioyXzRB2RtU8KVE8=
+golang.org/x/net v0.25.0 h1:d/OCCoBEUq33pjydKrGQhw7IlUPI2Oylr+8qLx49kac=
+golang.org/x/net v0.25.0/go.mod h1:JkAGAh7GEvH74S6FOH42FLoXpXbE/aqXSrIQjXgsiwM=
golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U=
golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw=
golang.org/x/oauth2 v0.0.0-20190604053449-0f29369cfe45/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw=
@@ -1388,8 +1388,8 @@ golang.org/x/sys v0.10.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.11.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.15.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
golang.org/x/sys v0.17.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
-golang.org/x/sys v0.19.0 h1:q5f1RH2jigJ1MoAWp2KTp3gm5zAGFUTarQZ5U386+4o=
-golang.org/x/sys v0.19.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
+golang.org/x/sys v0.20.0 h1:Od9JTbYCk261bKm4M/mw7AklTlFYIa0bIp9BgSm1S8Y=
+golang.org/x/sys v0.20.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
golang.org/x/term v0.0.0-20201117132131-f5c789dd3221/go.mod h1:Nr5EML6q2oocZ2LXRh80K7BxOlk5/8JxuGnuhpl+muw=
golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo=
golang.org/x/term v0.0.0-20210220032956-6a3ed077a48d/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo=
@@ -1401,8 +1401,8 @@ golang.org/x/term v0.7.0/go.mod h1:P32HKFT3hSsZrRxla30E9HqToFYAQPCMs/zFMBUFqPY=
golang.org/x/term v0.8.0/go.mod h1:xPskH00ivmX89bAKVGSKKtLOWNx2+17Eiy94tnKShWo=
golang.org/x/term v0.10.0/go.mod h1:lpqdcUyK/oCiQxvxVrppt5ggO2KCZ5QblwqPnfZ6d5o=
golang.org/x/term v0.15.0/go.mod h1:BDl952bC7+uMoWR75FIrCDx79TPU9oHkTZ9yRbYOrX0=
-golang.org/x/term v0.19.0 h1:+ThwsDv+tYfnJFhF4L8jITxu1tdTWRTZpdsWgEgjL6Q=
-golang.org/x/term v0.19.0/go.mod h1:2CuTdWZ7KHSQwUzKva0cbMg6q2DMI3Mmxp+gKJbskEk=
+golang.org/x/term v0.20.0 h1:VnkxpohqXaOBYJtBmEppKUG6mXpi+4O6purfc2+sMhw=
+golang.org/x/term v0.20.0/go.mod h1:8UkIAJTvZgivsXaD6/pH6U9ecQzZ45awqEOzuCvwpFY=
golang.org/x/text v0.0.0-20170915032832-14c0d48ead0c/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
golang.org/x/text v0.3.1-0.20180807135948-17ff2d5776d2/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
@@ -1417,8 +1417,9 @@ golang.org/x/text v0.7.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8=
golang.org/x/text v0.8.0/go.mod h1:e1OnstbJyHTd6l/uOt8jFFHp6TRDWZR/bV3emEE/zU8=
golang.org/x/text v0.9.0/go.mod h1:e1OnstbJyHTd6l/uOt8jFFHp6TRDWZR/bV3emEE/zU8=
golang.org/x/text v0.11.0/go.mod h1:TvPlkZtksWOMsz7fbANvkp4WM8x/WCo/om8BMLbz+aE=
-golang.org/x/text v0.14.0 h1:ScX5w1eTa3QqT8oi6+ziP7dTV1S2+ALU0bI+0zXKWiQ=
golang.org/x/text v0.14.0/go.mod h1:18ZOQIKpY8NJVqYksKHtTdi31H5itFRjB5/qKTNYzSU=
+golang.org/x/text v0.15.0 h1:h1V/4gjBv8v9cjcR6+AR5+/cIYK5N/WAgiv4xlsEtAk=
+golang.org/x/text v0.15.0/go.mod h1:18ZOQIKpY8NJVqYksKHtTdi31H5itFRjB5/qKTNYzSU=
golang.org/x/time v0.0.0-20180412165947-fbb02b2291d2/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
golang.org/x/time v0.0.0-20181108054448-85acf8d2951c/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
golang.org/x/time v0.0.0-20190308202827-9d24e82272b4/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
diff --git a/lib/tests/sdk_test.go b/lib/tests/sdk_test.go
index ee688d797e..b20c163d1a 100644
--- a/lib/tests/sdk_test.go
+++ b/lib/tests/sdk_test.go
@@ -29,7 +29,7 @@ func TestSimpleNuclei(t *testing.T) {
goleak.VerifyNone(t, knownLeaks...)
}()
ne, err := nuclei.NewNucleiEngine(
- nuclei.WithTemplateFilters(nuclei.TemplateFilters{ProtocolTypes: "dns"}),
+ nuclei.WithTemplateFilters(nuclei.TemplateFilters{ProtocolTypes: "dns"}), // filter dns templates
nuclei.EnableStatsWithOpts(nuclei.StatsOptions{JSON: true}),
)
require.Nil(t, err)
diff --git a/pkg/js/libs/postgres/postgres.go b/pkg/js/libs/postgres/postgres.go
index 3b52604f39..5aea415010 100644
--- a/pkg/js/libs/postgres/postgres.go
+++ b/pkg/js/libs/postgres/postgres.go
@@ -104,6 +104,7 @@ func executeQuery(host string, port int, username string, password string, dbNam
if err != nil {
return nil, err
}
+ defer db.Close()
rows, err := db.Query(query)
if err != nil {
@@ -143,12 +144,16 @@ func connect(host string, port int, username string, password string, dbName str
target := net.JoinHostPort(host, fmt.Sprintf("%d", port))
+ ctx, cancel := context.WithCancel(context.Background())
+ defer cancel()
+
db := pg.Connect(&pg.Options{
- Addr: target,
- User: username,
- Password: password,
- Database: dbName,
- })
+ Addr: target,
+ User: username,
+ Password: password,
+ Database: dbName,
+ IdleCheckFrequency: -1,
+ }).WithContext(ctx).WithTimeout(10 * time.Second)
defer db.Close()
_, err := db.Exec("select 1")
diff --git a/pkg/protocols/http/httputils/spm.go b/pkg/protocols/http/httputils/spm.go
index bb0b337c77..22fdd8f93b 100644
--- a/pkg/protocols/http/httputils/spm.go
+++ b/pkg/protocols/http/httputils/spm.go
@@ -123,9 +123,11 @@ func (h *StopAtFirstMatchHandler[T]) MatchCallback(fn func()) {
// run runs the internal handler
func (h *StopAtFirstMatchHandler[T]) run(ctx context.Context) {
defer h.internalWg.Done()
+
for {
select {
case <-ctx.Done():
+ return
case val, ok := <-h.ResultChan:
if !ok {
return
diff --git a/pkg/protocols/http/request.go b/pkg/protocols/http/request.go
index df914a785c..c372e3262b 100644
--- a/pkg/protocols/http/request.go
+++ b/pkg/protocols/http/request.go
@@ -114,7 +114,13 @@ func (request *Request) executeRaceRequest(input *contextargs.Context, previous
}
shouldStop := (request.options.Options.StopAtFirstMatch || request.StopAtFirstMatch || request.options.StopAtFirstMatch)
- spmHandler := httputils.NewNonBlockingSPMHandler[error](ctx, maxErrorsWhenParallel, shouldStop)
+
+ childCtx, cancel := context.WithCancel(context.Background())
+ defer cancel()
+
+ spmHandler := httputils.NewNonBlockingSPMHandler[error](childCtx, maxErrorsWhenParallel, shouldStop)
+ defer spmHandler.Cancel()
+
gotMatches := &atomic.Bool{}
// wrappedCallback is a callback that wraps the original callback
// to implement stop at first match logic
@@ -195,7 +201,13 @@ func (request *Request) executeParallelHTTP(input *contextargs.Context, dynamicV
// Stop-at-first-match logic while executing requests
// parallely using threads
shouldStop := (request.options.Options.StopAtFirstMatch || request.StopAtFirstMatch || request.options.StopAtFirstMatch)
- spmHandler := httputils.NewBlockingSPMHandler[error](context.Background(), maxWorkers, maxErrorsWhenParallel, shouldStop)
+
+ ctx, cancel := context.WithCancel(context.Background())
+ defer cancel()
+
+ spmHandler := httputils.NewBlockingSPMHandler[error](ctx, maxWorkers, maxErrorsWhenParallel, shouldStop)
+ defer spmHandler.Cancel()
+
// wrappedCallback is a callback that wraps the original callback
// to implement stop at first match logic
wrappedCallback := func(event *output.InternalWrappedEvent) {
@@ -327,7 +339,13 @@ func (request *Request) executeTurboHTTP(input *contextargs.Context, dynamicValu
// Stop-at-first-match logic while executing requests
// parallely using threads
shouldStop := (request.options.Options.StopAtFirstMatch || request.StopAtFirstMatch || request.options.StopAtFirstMatch)
- spmHandler := httputils.NewBlockingSPMHandler[error](context.Background(), maxWorkers, maxErrorsWhenParallel, shouldStop)
+
+ ctx, cancel := context.WithCancel(context.Background())
+ defer cancel()
+
+ spmHandler := httputils.NewBlockingSPMHandler[error](ctx, maxWorkers, maxErrorsWhenParallel, shouldStop)
+ defer spmHandler.Cancel()
+
// wrappedCallback is a callback that wraps the original callback
// to implement stop at first match logic
wrappedCallback := func(event *output.InternalWrappedEvent) {
diff --git a/pkg/utils/http_probe.go b/pkg/utils/http_probe.go
index 1ee4657039..9a0877f187 100644
--- a/pkg/utils/http_probe.go
+++ b/pkg/utils/http_probe.go
@@ -29,6 +29,7 @@ func ProbeURL(input string, httpxclient *httpx.HTTPX) string {
if _, err = httpxclient.Do(req, httpx.UnsafeOptions{}); err != nil {
continue
}
+
return formedURL
}
return ""
From bfddce4fbb7112f2dce1c1df25fef1d966cd059b Mon Sep 17 00:00:00 2001
From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com>
Date: Sun, 19 May 2024 19:19:49 +0000
Subject: [PATCH 013/283] chore(deps): bump
github.com/projectdiscovery/useragent
Bumps [github.com/projectdiscovery/useragent](https://github.com/projectdiscovery/useragent) from 0.0.49 to 0.0.52.
- [Release notes](https://github.com/projectdiscovery/useragent/releases)
- [Commits](https://github.com/projectdiscovery/useragent/compare/v0.0.49...v0.0.52)
---
updated-dependencies:
- dependency-name: github.com/projectdiscovery/useragent
dependency-type: direct:production
update-type: version-update:semver-patch
...
Signed-off-by: dependabot[bot]
---
go.mod | 3 +--
go.sum | 5 ++---
2 files changed, 3 insertions(+), 5 deletions(-)
diff --git a/go.mod b/go.mod
index 36d655cd1b..9516a8bee9 100644
--- a/go.mod
+++ b/go.mod
@@ -93,7 +93,7 @@ require (
github.com/projectdiscovery/sarif v0.0.1
github.com/projectdiscovery/tlsx v1.1.6
github.com/projectdiscovery/uncover v1.0.7
- github.com/projectdiscovery/useragent v0.0.49
+ github.com/projectdiscovery/useragent v0.0.52
github.com/projectdiscovery/utils v0.0.93
github.com/projectdiscovery/wappalyzergo v0.1.0
github.com/redis/go-redis/v9 v9.1.0
@@ -203,7 +203,6 @@ require (
github.com/projectdiscovery/freeport v0.0.5 // indirect
github.com/projectdiscovery/ldapserver v1.0.2-0.20240219154113-dcc758ebc0cb // indirect
github.com/projectdiscovery/machineid v0.0.0-20240226150047-2e2c51e35983 // indirect
- github.com/projectdiscovery/stringsutil v0.0.2 // indirect
github.com/quic-go/quic-go v0.42.0 // indirect
github.com/refraction-networking/utls v1.6.1 // indirect
github.com/sashabaranov/go-openai v1.15.3 // indirect
diff --git a/go.sum b/go.sum
index 2a8f67849f..a6542ad9b5 100644
--- a/go.sum
+++ b/go.sum
@@ -887,8 +887,8 @@ github.com/projectdiscovery/tlsx v1.1.6 h1:iw2zwKbd2+kRQ8J1G4dLmS0CLyemd/tKz1Uzc
github.com/projectdiscovery/tlsx v1.1.6/go.mod h1:s7SRRFdrwIZBK/RXXZi4CR/CubqFSvp8h5Bk1srEZIo=
github.com/projectdiscovery/uncover v1.0.7 h1:ut+2lTuvmftmveqF5RTjMWAgyLj8ltPQC7siFy9sj0A=
github.com/projectdiscovery/uncover v1.0.7/go.mod h1:HFXgm1sRPuoN0D4oATljPIdmbo/EEh1wVuxQqo/dwFE=
-github.com/projectdiscovery/useragent v0.0.49 h1:wQc9i+Xy+mUMJ45Ralv1JsQImRWqEOEvpYUe6MchScg=
-github.com/projectdiscovery/useragent v0.0.49/go.mod h1:jQz6X/usiXrPYE6B/1uVKuzIrBJXgw9hLC9eeNy38+0=
+github.com/projectdiscovery/useragent v0.0.52 h1:9SUPH0Epo3DJfB6PCDgETfMaD6nZ08sFvfgXTmPUAsU=
+github.com/projectdiscovery/useragent v0.0.52/go.mod h1:PUXHgShvaD8p3bihy1mY8tuBDhdk3M0yy4Z10Ajg2yQ=
github.com/projectdiscovery/utils v0.0.93 h1:IMZFsmQFYZUf7rxpBoZj+53FsNDC/vHsXA+4B4GuGeg=
github.com/projectdiscovery/utils v0.0.93/go.mod h1:2+mWzk5FeYdK9imo5eLk6oVeih0G0wsTff1pzBAh9tk=
github.com/projectdiscovery/wappalyzergo v0.1.0 h1:ZagOIKemBsNfCDRQeWavWEXtEjP8UiziuoRoPTwDAJ0=
@@ -949,7 +949,6 @@ github.com/rs/xid v1.5.0 h1:mKX4bl4iPYJtEIxp6CYiUuLQ/8DYMoz0PUdtGgMFRVc=
github.com/rs/xid v1.5.0/go.mod h1:trrq9SKmegXys3aeAKXMUTdJsYXVwGY3RLcfgqegfbg=
github.com/russross/blackfriday/v2 v2.0.1/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM=
github.com/ryanuber/columnize v0.0.0-20160712163229-9b3edd62028f/go.mod h1:sm1tb6uqfes/u+d4ooFouqFdy9/2g9QGwK3SQygK0Ts=
-github.com/saintfish/chardet v0.0.0-20120816061221-3af4cd4741ca/go.mod h1:uugorj2VCxiV1x+LzaIdVa9b4S4qGAcH6cbhh4qVxOU=
github.com/saintfish/chardet v0.0.0-20230101081208-5e3ef4b5456d h1:hrujxIzL1woJ7AwssoOcM/tq5JjjG2yYOc8odClEiXA=
github.com/saintfish/chardet v0.0.0-20230101081208-5e3ef4b5456d/go.mod h1:uugorj2VCxiV1x+LzaIdVa9b4S4qGAcH6cbhh4qVxOU=
github.com/samuel/go-zookeeper v0.0.0-20190923202752-2cc03de413da/go.mod h1:gi+0XIa01GRL2eRQVjQkKGqKF3SF9vZR/HnPullcV2E=
From 9cc335a34cc9e29ffbc0991003bae9b87514709c Mon Sep 17 00:00:00 2001
From: Mzack9999
Date: Sun, 19 May 2024 23:12:21 +0200
Subject: [PATCH 014/283] first interface ipv4 address (#5186)
---
pkg/protocols/common/protocolstate/state.go | 1 +
1 file changed, 1 insertion(+)
diff --git a/pkg/protocols/common/protocolstate/state.go b/pkg/protocols/common/protocolstate/state.go
index 67820ec074..ae5ac6cb9d 100644
--- a/pkg/protocols/common/protocolstate/state.go
+++ b/pkg/protocols/common/protocolstate/state.go
@@ -180,6 +180,7 @@ func interfaceAddress(interfaceName string) (net.IP, error) {
if ipnet, ok := addr.(*net.IPNet); ok && !ipnet.IP.IsLoopback() {
if ipnet.IP.To4() != nil {
address = ipnet.IP
+ break
}
}
}
From 48ed1dde38398e01122dc80db76d80eea3075092 Mon Sep 17 00:00:00 2001
From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com>
Date: Mon, 20 May 2024 05:06:38 +0000
Subject: [PATCH 015/283] chore(deps): bump github.com/projectdiscovery/utils
Bumps [github.com/projectdiscovery/utils](https://github.com/projectdiscovery/utils) from 0.0.93 to 0.0.94.
- [Release notes](https://github.com/projectdiscovery/utils/releases)
- [Changelog](https://github.com/projectdiscovery/utils/blob/main/CHANGELOG.md)
- [Commits](https://github.com/projectdiscovery/utils/compare/v0.0.93...v0.0.94)
---
updated-dependencies:
- dependency-name: github.com/projectdiscovery/utils
dependency-type: direct:production
update-type: version-update:semver-patch
...
Signed-off-by: dependabot[bot]
---
go.mod | 2 +-
go.sum | 4 ++--
2 files changed, 3 insertions(+), 3 deletions(-)
diff --git a/go.mod b/go.mod
index 9516a8bee9..25e4f38a33 100644
--- a/go.mod
+++ b/go.mod
@@ -94,7 +94,7 @@ require (
github.com/projectdiscovery/tlsx v1.1.6
github.com/projectdiscovery/uncover v1.0.7
github.com/projectdiscovery/useragent v0.0.52
- github.com/projectdiscovery/utils v0.0.93
+ github.com/projectdiscovery/utils v0.0.94
github.com/projectdiscovery/wappalyzergo v0.1.0
github.com/redis/go-redis/v9 v9.1.0
github.com/seh-msft/burpxml v1.0.1
diff --git a/go.sum b/go.sum
index a6542ad9b5..151484cfb3 100644
--- a/go.sum
+++ b/go.sum
@@ -889,8 +889,8 @@ github.com/projectdiscovery/uncover v1.0.7 h1:ut+2lTuvmftmveqF5RTjMWAgyLj8ltPQC7
github.com/projectdiscovery/uncover v1.0.7/go.mod h1:HFXgm1sRPuoN0D4oATljPIdmbo/EEh1wVuxQqo/dwFE=
github.com/projectdiscovery/useragent v0.0.52 h1:9SUPH0Epo3DJfB6PCDgETfMaD6nZ08sFvfgXTmPUAsU=
github.com/projectdiscovery/useragent v0.0.52/go.mod h1:PUXHgShvaD8p3bihy1mY8tuBDhdk3M0yy4Z10Ajg2yQ=
-github.com/projectdiscovery/utils v0.0.93 h1:IMZFsmQFYZUf7rxpBoZj+53FsNDC/vHsXA+4B4GuGeg=
-github.com/projectdiscovery/utils v0.0.93/go.mod h1:2+mWzk5FeYdK9imo5eLk6oVeih0G0wsTff1pzBAh9tk=
+github.com/projectdiscovery/utils v0.0.94 h1:2zzFEjMkq/Ei/o3NIA2SWTkhfGHMkBy0T3aIzq0vizo=
+github.com/projectdiscovery/utils v0.0.94/go.mod h1:wxPi+kCsLm5JCLMkZJyGwS+4Mn4PaPHHf0ayE8JphOw=
github.com/projectdiscovery/wappalyzergo v0.1.0 h1:ZagOIKemBsNfCDRQeWavWEXtEjP8UiziuoRoPTwDAJ0=
github.com/projectdiscovery/wappalyzergo v0.1.0/go.mod h1:wBYGKmA5BQp/NWsAy1q/jSH8N1LHWQ/LV26DuR+KzPM=
github.com/projectdiscovery/yamldoc-go v1.0.4 h1:eZoESapnMw6WAHiVgRwNqvbJEfNHEH148uthhFbG5jE=
From e92746ddcda9458bffbadbca16183f0dec9472f0 Mon Sep 17 00:00:00 2001
From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com>
Date: Mon, 20 May 2024 05:06:44 +0000
Subject: [PATCH 016/283] chore(deps): bump github.com/projectdiscovery/rawhttp
Bumps [github.com/projectdiscovery/rawhttp](https://github.com/projectdiscovery/rawhttp) from 0.1.49 to 0.1.50.
- [Release notes](https://github.com/projectdiscovery/rawhttp/releases)
- [Commits](https://github.com/projectdiscovery/rawhttp/compare/v0.1.49...v0.1.50)
---
updated-dependencies:
- dependency-name: github.com/projectdiscovery/rawhttp
dependency-type: direct:production
update-type: version-update:semver-patch
...
Signed-off-by: dependabot[bot]
---
go.mod | 2 +-
go.sum | 4 ++--
2 files changed, 3 insertions(+), 3 deletions(-)
diff --git a/go.mod b/go.mod
index 9516a8bee9..1c8cb6965b 100644
--- a/go.mod
+++ b/go.mod
@@ -23,7 +23,7 @@ require (
github.com/projectdiscovery/fastdialer v0.0.71
github.com/projectdiscovery/hmap v0.0.42
github.com/projectdiscovery/interactsh v1.1.10-0.20240519152711-6a0cb98b1663
- github.com/projectdiscovery/rawhttp v0.1.49
+ github.com/projectdiscovery/rawhttp v0.1.50
github.com/projectdiscovery/retryabledns v1.0.59
github.com/projectdiscovery/retryablehttp-go v1.0.60
github.com/projectdiscovery/yamldoc-go v1.0.4
diff --git a/go.sum b/go.sum
index a6542ad9b5..5042b9b1bc 100644
--- a/go.sum
+++ b/go.sum
@@ -871,8 +871,8 @@ github.com/projectdiscovery/networkpolicy v0.0.8 h1:XvfBaBwSDNTesSfNQP9VLk3HX9I7
github.com/projectdiscovery/networkpolicy v0.0.8/go.mod h1:xnjNqhemxUPxU+UD5Jgsc3+K8IVmcqT1SJeo6UzMtkI=
github.com/projectdiscovery/ratelimit v0.0.40 h1:SURkc7+ezUeGNt9PEQZOtd7nh2o2jKzmqcehrrf6DMU=
github.com/projectdiscovery/ratelimit v0.0.40/go.mod h1:3Z1rK0bs4BSjdYNvIlbmZR1H8Z5MsXRz+QRCXGIB0HE=
-github.com/projectdiscovery/rawhttp v0.1.49 h1:OPP9R/UZx/GFrcPRUs9fOS1dLwhg+2o7p3dByzkIhWM=
-github.com/projectdiscovery/rawhttp v0.1.49/go.mod h1:aaAaMsdzHPfw4yU3nbeP7NI3vy1ZjgoXw7l+m4Tnt94=
+github.com/projectdiscovery/rawhttp v0.1.50 h1:JQw0jBnUEcU2ZoTi8TBQpxHKykxkoBSfh9WETwggZIM=
+github.com/projectdiscovery/rawhttp v0.1.50/go.mod h1:h3PiVqB8w7U/yOK+TCAJb1/zu0FHPCFM2ZQ1k2qm9cg=
github.com/projectdiscovery/rdap v0.9.1-0.20221108103045-9865884d1917 h1:m03X4gBVSorSzvmm0bFa7gDV4QNSOWPL/fgZ4kTXBxk=
github.com/projectdiscovery/rdap v0.9.1-0.20221108103045-9865884d1917/go.mod h1:JxXtZC9e195awe7EynrcnBJmFoad/BNDzW9mzFkK8Sg=
github.com/projectdiscovery/retryabledns v1.0.59 h1:8pMN+VibEBp29RIUior9LXUbx0RsBTjPC0008t2hfGU=
From 8cffeecd4f23ffdee0e8949ef3b4efe56e7c1c08 Mon Sep 17 00:00:00 2001
From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com>
Date: Mon, 20 May 2024 05:33:04 +0000
Subject: [PATCH 017/283] chore(deps): bump
github.com/projectdiscovery/ratelimit
Bumps [github.com/projectdiscovery/ratelimit](https://github.com/projectdiscovery/ratelimit) from 0.0.40 to 0.0.42.
- [Release notes](https://github.com/projectdiscovery/ratelimit/releases)
- [Commits](https://github.com/projectdiscovery/ratelimit/compare/v0.0.40...v0.0.42)
---
updated-dependencies:
- dependency-name: github.com/projectdiscovery/ratelimit
dependency-type: direct:production
update-type: version-update:semver-patch
...
Signed-off-by: dependabot[bot]
---
go.mod | 2 +-
go.sum | 4 ++--
2 files changed, 3 insertions(+), 3 deletions(-)
diff --git a/go.mod b/go.mod
index ba2b86d801..7143ae29fe 100644
--- a/go.mod
+++ b/go.mod
@@ -88,7 +88,7 @@ require (
github.com/projectdiscovery/httpx v1.6.1
github.com/projectdiscovery/mapcidr v1.1.34
github.com/projectdiscovery/n3iwf v0.0.0-20230523120440-b8cd232ff1f5
- github.com/projectdiscovery/ratelimit v0.0.40
+ github.com/projectdiscovery/ratelimit v0.0.42
github.com/projectdiscovery/rdap v0.9.1-0.20221108103045-9865884d1917
github.com/projectdiscovery/sarif v0.0.1
github.com/projectdiscovery/tlsx v1.1.6
diff --git a/go.sum b/go.sum
index b2fc867f65..70d8985d60 100644
--- a/go.sum
+++ b/go.sum
@@ -869,8 +869,8 @@ github.com/projectdiscovery/n3iwf v0.0.0-20230523120440-b8cd232ff1f5 h1:L/e8z8yw
github.com/projectdiscovery/n3iwf v0.0.0-20230523120440-b8cd232ff1f5/go.mod h1:pGW2ncnTxTxHtP9wzcIJAB+3/NMp6IiuQWd2NK7K+oc=
github.com/projectdiscovery/networkpolicy v0.0.8 h1:XvfBaBwSDNTesSfNQP9VLk3HX9I7x7gHm028TJ5XwI8=
github.com/projectdiscovery/networkpolicy v0.0.8/go.mod h1:xnjNqhemxUPxU+UD5Jgsc3+K8IVmcqT1SJeo6UzMtkI=
-github.com/projectdiscovery/ratelimit v0.0.40 h1:SURkc7+ezUeGNt9PEQZOtd7nh2o2jKzmqcehrrf6DMU=
-github.com/projectdiscovery/ratelimit v0.0.40/go.mod h1:3Z1rK0bs4BSjdYNvIlbmZR1H8Z5MsXRz+QRCXGIB0HE=
+github.com/projectdiscovery/ratelimit v0.0.42 h1:110tBLaGUgyPK0DjLmMBQaaqW4tkmqsFidr/t6tyOy4=
+github.com/projectdiscovery/ratelimit v0.0.42/go.mod h1:ruhLiZ5liukSpG07p6eHTCPJUmTwOhxDSxQPulvC3/Y=
github.com/projectdiscovery/rawhttp v0.1.50 h1:JQw0jBnUEcU2ZoTi8TBQpxHKykxkoBSfh9WETwggZIM=
github.com/projectdiscovery/rawhttp v0.1.50/go.mod h1:h3PiVqB8w7U/yOK+TCAJb1/zu0FHPCFM2ZQ1k2qm9cg=
github.com/projectdiscovery/rdap v0.9.1-0.20221108103045-9865884d1917 h1:m03X4gBVSorSzvmm0bFa7gDV4QNSOWPL/fgZ4kTXBxk=
From f266542dd457b168fb03cdebb969b23a75930ffc Mon Sep 17 00:00:00 2001
From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com>
Date: Mon, 20 May 2024 05:33:16 +0000
Subject: [PATCH 018/283] chore(deps): bump
github.com/projectdiscovery/fastdialer
Bumps [github.com/projectdiscovery/fastdialer](https://github.com/projectdiscovery/fastdialer) from 0.0.71 to 0.0.72.
- [Release notes](https://github.com/projectdiscovery/fastdialer/releases)
- [Commits](https://github.com/projectdiscovery/fastdialer/compare/v0.0.71...v0.0.72)
---
updated-dependencies:
- dependency-name: github.com/projectdiscovery/fastdialer
dependency-type: direct:production
update-type: version-update:semver-patch
...
Signed-off-by: dependabot[bot]
---
go.mod | 2 +-
go.sum | 4 ++--
2 files changed, 3 insertions(+), 3 deletions(-)
diff --git a/go.mod b/go.mod
index ba2b86d801..13a195a3ba 100644
--- a/go.mod
+++ b/go.mod
@@ -20,7 +20,7 @@ require (
github.com/olekukonko/tablewriter v0.0.5
github.com/pkg/errors v0.9.1
github.com/projectdiscovery/clistats v0.0.20
- github.com/projectdiscovery/fastdialer v0.0.71
+ github.com/projectdiscovery/fastdialer v0.0.72
github.com/projectdiscovery/hmap v0.0.42
github.com/projectdiscovery/interactsh v1.1.10-0.20240519152711-6a0cb98b1663
github.com/projectdiscovery/rawhttp v0.1.50
diff --git a/go.sum b/go.sum
index b2fc867f65..f05fdf4020 100644
--- a/go.sum
+++ b/go.sum
@@ -837,8 +837,8 @@ github.com/projectdiscovery/clistats v0.0.20 h1:5jO5SLiRJ7f0nDV0ndBNmBeesbROouPo
github.com/projectdiscovery/clistats v0.0.20/go.mod h1:GJ2av0KnOvK0AISQnP8hyDclYIji1LVkx2l0pwnzAu4=
github.com/projectdiscovery/dsl v0.0.56 h1:iVFIfDdGXkrXTh5sf6hRaxqTTEYiv/LnNjoOHJIfHiY=
github.com/projectdiscovery/dsl v0.0.56/go.mod h1:3UBjPvtiy8c5E8oAUrKobMKjb3kEwQuGB5tqIN964Og=
-github.com/projectdiscovery/fastdialer v0.0.71 h1:96j6Y65hDPZ9AzlYpp95hvIH5Yx/0OE2UTx+frWfnm4=
-github.com/projectdiscovery/fastdialer v0.0.71/go.mod h1:b/oPPVSoLLD2N4W2/HrXbhQbyJVXqRw8CK1lenCUk64=
+github.com/projectdiscovery/fastdialer v0.0.72 h1:CbKNFqvJotGmn9uBeHoR2vJQRg8QMuQs9NIOc8HW02E=
+github.com/projectdiscovery/fastdialer v0.0.72/go.mod h1:sfeBKELnLnkpwEYcK5Qf8DRXLcdmR34u4TxtFwxNNQ0=
github.com/projectdiscovery/fasttemplate v0.0.2 h1:h2cISk5xDhlJEinlBQS6RRx0vOlOirB2y3Yu4PJzpiA=
github.com/projectdiscovery/fasttemplate v0.0.2/go.mod h1:XYWWVMxnItd+r0GbjA1GCsUopMw1/XusuQxdyAIHMCw=
github.com/projectdiscovery/freeport v0.0.5 h1:jnd3Oqsl4S8n0KuFkE5Hm8WGDP24ITBvmyw5pFTHS8Q=
From f6332583b7f62c063b3050db18059c786d2e8e0b Mon Sep 17 00:00:00 2001
From: Mzack9999
Date: Tue, 21 May 2024 18:12:01 +0200
Subject: [PATCH 019/283] making ssl errors non fatal (#5203)
* making ssl errors non fatal
* adding test
---
cmd/integration-test/ssl.go | 21 ++++++++++++
.../protocols/ssl/multi-req.yaml | 34 +++++++++++++++++++
pkg/tmplexec/multiproto/multi.go | 14 +++++++-
3 files changed, 68 insertions(+), 1 deletion(-)
create mode 100644 integration_tests/protocols/ssl/multi-req.yaml
diff --git a/cmd/integration-test/ssl.go b/cmd/integration-test/ssl.go
index 70b8c48445..24f6e9b903 100644
--- a/cmd/integration-test/ssl.go
+++ b/cmd/integration-test/ssl.go
@@ -13,6 +13,7 @@ var sslTestcases = []TestCaseInfo{
{Path: "protocols/ssl/custom-cipher.yaml", TestCase: &sslCustomCipher{}},
{Path: "protocols/ssl/custom-version.yaml", TestCase: &sslCustomVersion{}},
{Path: "protocols/ssl/ssl-with-vars.yaml", TestCase: &sslWithVars{}},
+ {Path: "protocols/ssl/multi-req.yaml", TestCase: &sslMultiReq{}},
}
type sslBasic struct{}
@@ -118,3 +119,23 @@ func (h *sslWithVars) Execute(filePath string) error {
return expectResultsCount(results, 1)
}
+
+type sslMultiReq struct{}
+
+func (h *sslMultiReq) Execute(filePath string) error {
+ ts := testutils.NewTCPServer(&tls.Config{}, defaultStaticPort, func(conn net.Conn) {
+ defer conn.Close()
+ data := make([]byte, 4)
+ if _, err := conn.Read(data); err != nil {
+ return
+ }
+ })
+ defer ts.Close()
+
+ results, err := testutils.RunNucleiTemplateAndGetResults(filePath, ts.URL, debug, "-V")
+ if err != nil {
+ return err
+ }
+
+ return expectResultsCount(results, 2)
+}
diff --git a/integration_tests/protocols/ssl/multi-req.yaml b/integration_tests/protocols/ssl/multi-req.yaml
new file mode 100644
index 0000000000..670ca633d9
--- /dev/null
+++ b/integration_tests/protocols/ssl/multi-req.yaml
@@ -0,0 +1,34 @@
+id: multi-req
+
+info:
+ name: Multi-Request
+ author: pdteam
+ severity: info
+
+ssl:
+ - address: "{{Host}}:{{Port}}"
+ min_version: ssl30
+ max_version: ssl30
+
+ extractors:
+ - type: json
+ json:
+ - " .tls_version"
+
+ - address: "{{Host}}:{{Port}}"
+ min_version: tls10
+ max_version: tls10
+
+ extractors:
+ - type: json
+ json:
+ - " .tls_version"
+
+ - address: "{{Host}}:{{Port}}"
+ min_version: tls11
+ max_version: tls11
+
+ extractors:
+ - type: json
+ json:
+ - " .tls_version"
\ No newline at end of file
diff --git a/pkg/tmplexec/multiproto/multi.go b/pkg/tmplexec/multiproto/multi.go
index 58858f971e..7bbc2a1403 100644
--- a/pkg/tmplexec/multiproto/multi.go
+++ b/pkg/tmplexec/multiproto/multi.go
@@ -8,6 +8,8 @@ import (
"github.com/projectdiscovery/nuclei/v3/pkg/protocols"
"github.com/projectdiscovery/nuclei/v3/pkg/protocols/common/generators"
"github.com/projectdiscovery/nuclei/v3/pkg/scan"
+ "github.com/projectdiscovery/nuclei/v3/pkg/templates/types"
+ stringsutil "github.com/projectdiscovery/utils/strings"
)
// Mutliprotocol is a template executer engine that executes multiple protocols
@@ -110,9 +112,19 @@ func (m *MultiProtocol) ExecuteWithResults(ctx *scan.ScanContext) error {
values := m.options.GetTemplateCtx(ctx.Input.MetaInput).GetAll()
err := req.ExecuteWithResults(ctx.Input, output.InternalEvent(values), nil, multiProtoCallback)
- // if error skip execution of next protocols
+ // in case of fatal error skip execution of next protocols
if err != nil {
+ // always log errors
ctx.LogError(err)
+
+ // for some classes of protocols (i.e ssl) errors like tls handshake are a legitimate behavior so we don't stop execution
+ // connection failures are already tracked by the internal host error cache
+ // we use strings comparison as the error is not formalized into instance within the standard library
+ // within a flow instead we consider ssl errors as fatal, since a specific logic was requested
+ if req.Type() == types.SSLProtocol && stringsutil.ContainsAnyI(err.Error(), "protocol version not supported", "could not do tls handshake") {
+ continue
+ }
+
return err
}
}
From 46e4810efb29d869cb66bca59c5dcd37e413183a Mon Sep 17 00:00:00 2001
From: mzack9999
Date: Wed, 22 May 2024 14:54:01 +0200
Subject: [PATCH 020/283] Fixing nil ptr on error
---
cmd/nuclei/main.go | 5 ++++-
1 file changed, 4 insertions(+), 1 deletion(-)
diff --git a/cmd/nuclei/main.go b/cmd/nuclei/main.go
index a1123b9ffc..e93a4c4441 100644
--- a/cmd/nuclei/main.go
+++ b/cmd/nuclei/main.go
@@ -141,7 +141,10 @@ func main() {
nucleiRunner.Close()
gologger.Info().Msgf("Creating resume file: %s\n", resumeFileName)
err := nucleiRunner.SaveResumeConfig(resumeFileName)
- return errorutil.NewWithErr(err).Msgf("couldn't create crash resume file")
+ if err != nil {
+ return errorutil.NewWithErr(err).Msgf("couldn't create crash resume file")
+ }
+ return nil
})
}
From 23bd0336fb4fa5ee6f522c1643458d2d7b0aa022 Mon Sep 17 00:00:00 2001
From: Tarun Koyalwar <45962551+tarunKoyalwar@users.noreply.github.com>
Date: Sat, 25 May 2024 00:29:04 +0530
Subject: [PATCH 021/283] multiple bug fixes + performance improvements (#5148)
* prototype errkit
* complete errkit implementation
* add cause to all timeouts
* fix request timeout annotation @timeout
* increase responseHeaderTimeout to 8 for stability
* rawhttp error related improvements
* feat: add port status caching
* add port status caching to http
* migrate to new utils/errkit
* remote dialinterface + error cause
* debug dir support using .gitignore debug-*
* make nuclei easy to debug
* debug dir update .gitignore
* temp change (to revert)
* Revert "temp change (to revert)"
This reverts commit d3131f777713b9f80e2275142e80f36340a76d36.
* use available context instead of new one
* bump fastdialer
* fix hosterrorscache + misc improvements
* add 'address' field in error log
* fix js vague errors + pgwrap driver
* fix max host error + misc updates
* update tests as per changes
* fix request annotation context
* remove closed dialer reference
* fix sdk panic issue
* bump retryablehttp-go,utils,fastdialer
---------
Co-authored-by: Sandeep Singh
---
.gitignore | 2 +-
go.mod | 45 ++++---
go.sum | 112 ++++++++--------
internal/runner/options.go | 8 +-
internal/runner/runner.go | 6 +-
lib/multi.go | 1 -
lib/sdk_private.go | 2 -
pkg/catalog/config/constants.go | 12 +-
pkg/catalog/config/nucleiconfig.go | 48 ++++++-
pkg/core/executors.go | 2 +-
pkg/core/workflow_execute.go | 2 +-
pkg/js/compiler/compiler.go | 11 +-
pkg/js/compiler/init.go | 9 +-
pkg/js/libs/postgres/memo.postgres.go | 2 +-
pkg/js/libs/postgres/postgres.go | 8 +-
pkg/js/utils/pgwrap/pgwrap.go | 53 ++++++++
pkg/output/output.go | 75 +++++++++--
pkg/output/output_test.go | 4 +-
pkg/protocols/code/code.go | 6 +-
.../common/contextargs/contextargs.go | 24 ++++
pkg/protocols/common/contextargs/metainput.go | 52 ++++++++
.../common/hosterrorscache/hosterrorscache.go | 120 +++++++++++++++---
.../hosterrorscache/hosterrorscache_test.go | 35 +++--
pkg/protocols/common/protocolinit/init.go | 2 +-
pkg/protocols/common/protocolstate/state.go | 1 +
pkg/protocols/headless/engine/page_actions.go | 5 +-
.../http/httpclientpool/clientpool.go | 27 +++-
pkg/protocols/http/httpclientpool/options.go | 54 ++++++++
pkg/protocols/http/request.go | 102 +++++++++++----
pkg/protocols/http/request_annotations.go | 19 ++-
pkg/protocols/http/request_fuzz.go | 7 +-
pkg/protocols/javascript/js.go | 28 ++--
pkg/protocols/network/request.go | 48 ++++++-
pkg/protocols/websocket/websocket.go | 3 +-
pkg/templates/cluster.go | 4 +-
pkg/testutils/testutils.go | 2 +
pkg/tmplexec/generic/exec.go | 2 +-
pkg/types/nucleierr/kinds.go | 28 ++++
38 files changed, 767 insertions(+), 204 deletions(-)
create mode 100644 pkg/js/utils/pgwrap/pgwrap.go
create mode 100644 pkg/protocols/http/httpclientpool/options.go
create mode 100644 pkg/types/nucleierr/kinds.go
diff --git a/.gitignore b/.gitignore
index 9386137b54..b9e561e3a6 100644
--- a/.gitignore
+++ b/.gitignore
@@ -39,5 +39,5 @@ integration_tests/fuzzplayground
/nuclei-stats
/nuclei-stats-*
/scan-charts
-
+/**/debug-*
diff --git a/go.mod b/go.mod
index e775ff43d7..537cc9ce65 100644
--- a/go.mod
+++ b/go.mod
@@ -16,16 +16,16 @@ require (
github.com/json-iterator/go v1.1.12
github.com/julienschmidt/httprouter v1.3.0
github.com/logrusorgru/aurora v2.0.3+incompatible
- github.com/miekg/dns v1.1.57
+ github.com/miekg/dns v1.1.59
github.com/olekukonko/tablewriter v0.0.5
github.com/pkg/errors v0.9.1
github.com/projectdiscovery/clistats v0.0.20
- github.com/projectdiscovery/fastdialer v0.0.72
- github.com/projectdiscovery/hmap v0.0.42
- github.com/projectdiscovery/interactsh v1.1.10-0.20240519152711-6a0cb98b1663
- github.com/projectdiscovery/rawhttp v0.1.50
- github.com/projectdiscovery/retryabledns v1.0.59
- github.com/projectdiscovery/retryablehttp-go v1.0.60
+ github.com/projectdiscovery/fastdialer v0.1.0
+ github.com/projectdiscovery/hmap v0.0.43
+ github.com/projectdiscovery/interactsh v1.1.9
+ github.com/projectdiscovery/rawhttp v0.1.49
+ github.com/projectdiscovery/retryabledns v1.0.60
+ github.com/projectdiscovery/retryablehttp-go v1.0.62
github.com/projectdiscovery/yamldoc-go v1.0.4
github.com/remeh/sizedwaitgroup v1.0.0
github.com/rs/xid v1.5.0
@@ -35,7 +35,7 @@ require (
github.com/spf13/cast v1.5.1
github.com/syndtr/goleveldb v1.0.0
github.com/valyala/fasttemplate v1.2.2
- github.com/weppos/publicsuffix-go v0.30.2-0.20230730094716-a20f9abcc222
+ github.com/weppos/publicsuffix-go v0.30.2
github.com/xanzy/go-gitlab v0.84.0
go.uber.org/multierr v1.11.0
golang.org/x/net v0.25.0
@@ -73,7 +73,7 @@ require (
github.com/h2non/filetype v1.1.3
github.com/labstack/echo/v4 v4.10.2
github.com/leslie-qiwa/flat v0.0.0-20230424180412-f9d1cf014baa
- github.com/lib/pq v1.10.1
+ github.com/lib/pq v1.10.9
github.com/mattn/go-sqlite3 v1.14.22
github.com/mholt/archiver v3.1.1+incompatible
github.com/ory/dockertest/v3 v3.10.0
@@ -94,12 +94,12 @@ require (
github.com/projectdiscovery/tlsx v1.1.6
github.com/projectdiscovery/uncover v1.0.7
github.com/projectdiscovery/useragent v0.0.52
- github.com/projectdiscovery/utils v0.0.94
+ github.com/projectdiscovery/utils v0.1.0
github.com/projectdiscovery/wappalyzergo v0.1.0
github.com/redis/go-redis/v9 v9.1.0
github.com/seh-msft/burpxml v1.0.1
github.com/stretchr/testify v1.9.0
- github.com/tarunKoyalwar/goleak v0.0.0-20240426214851-746d64600adc
+ github.com/tarunKoyalwar/goleak v0.0.0-20240429141123-0efa90dbdcf9
github.com/zmap/zgrab2 v0.1.8-0.20230806160807-97ba87c0e706
golang.org/x/term v0.20.0
gopkg.in/yaml.v3 v3.0.1
@@ -134,7 +134,7 @@ require (
github.com/cheggaaa/pb/v3 v3.1.4 // indirect
github.com/chenzhuoyu/base64x v0.0.0-20221115062448-fe3a3abad311 // indirect
github.com/cloudflare/cfssl v1.6.4 // indirect
- github.com/cloudflare/circl v1.3.7 // indirect
+ github.com/cloudflare/circl v1.3.8 // indirect
github.com/containerd/continuity v0.4.2 // indirect
github.com/cyphar/filepath-securejoin v0.2.4 // indirect
github.com/davidmz/go-pageant v1.0.2 // indirect
@@ -175,7 +175,7 @@ require (
github.com/jmespath/go-jmespath v0.4.0 // indirect
github.com/josharian/intern v1.0.0 // indirect
github.com/kataras/jwt v0.1.10 // indirect
- github.com/klauspost/compress v1.17.6 // indirect
+ github.com/klauspost/compress v1.17.8 // indirect
github.com/klauspost/pgzip v1.2.6 // indirect
github.com/kylelemons/godebug v1.1.0 // indirect
github.com/logrusorgru/aurora/v4 v4.0.0 // indirect
@@ -203,16 +203,15 @@ require (
github.com/projectdiscovery/freeport v0.0.5 // indirect
github.com/projectdiscovery/ldapserver v1.0.2-0.20240219154113-dcc758ebc0cb // indirect
github.com/projectdiscovery/machineid v0.0.0-20240226150047-2e2c51e35983 // indirect
- github.com/quic-go/quic-go v0.42.0 // indirect
- github.com/refraction-networking/utls v1.6.1 // indirect
+ github.com/refraction-networking/utls v1.6.6 // indirect
github.com/sashabaranov/go-openai v1.15.3 // indirect
github.com/shirou/gopsutil v3.21.11+incompatible // indirect
github.com/shoenig/go-m1cpu v0.1.6 // indirect
github.com/sirupsen/logrus v1.9.3 // indirect
github.com/skeema/knownhosts v1.2.1 // indirect
github.com/tidwall/btree v1.7.0 // indirect
- github.com/tidwall/buntdb v1.3.0 // indirect
- github.com/tidwall/gjson v1.17.0 // indirect
+ github.com/tidwall/buntdb v1.3.1 // indirect
+ github.com/tidwall/gjson v1.17.1 // indirect
github.com/tidwall/grect v0.1.4 // indirect
github.com/tidwall/match v1.1.1 // indirect
github.com/tidwall/pretty v1.2.1 // indirect
@@ -233,7 +232,7 @@ require (
github.com/zeebo/blake3 v0.2.3 // indirect
go.uber.org/goleak v1.3.0 // indirect
golang.org/x/arch v0.3.0 // indirect
- golang.org/x/sync v0.6.0 // indirect
+ golang.org/x/sync v0.7.0 // indirect
gopkg.in/djherbis/times.v1 v1.3.0 // indirect
mellium.im/sasl v0.3.1 // indirect
)
@@ -298,16 +297,16 @@ require (
github.com/ysmood/leakless v0.8.0 // indirect
github.com/yusufpapurcu/wmi v1.2.4 // indirect
github.com/zmap/rc2 v0.0.0-20190804163417-abaa70531248 // indirect
- github.com/zmap/zcrypto v0.0.0-20231219022726-a1f61fb1661c // indirect
- go.etcd.io/bbolt v1.3.8 // indirect
+ github.com/zmap/zcrypto v0.0.0-20240512203510-0fef58d9a9db // indirect
+ go.etcd.io/bbolt v1.3.10 // indirect
go.uber.org/zap v1.25.0 // indirect
goftp.io/server/v2 v2.0.1 // indirect
golang.org/x/crypto v0.23.0 // indirect
- golang.org/x/exp v0.0.0-20240119083558-1b970713d09a
- golang.org/x/mod v0.14.0 // indirect
+ golang.org/x/exp v0.0.0-20240506185415-9bf2ced13842
+ golang.org/x/mod v0.17.0 // indirect
golang.org/x/sys v0.20.0 // indirect
golang.org/x/time v0.5.0 // indirect
- golang.org/x/tools v0.17.0
+ golang.org/x/tools v0.21.0
google.golang.org/appengine v1.6.7 // indirect
google.golang.org/protobuf v1.33.0 // indirect
gopkg.in/alecthomas/kingpin.v2 v2.2.6 // indirect
diff --git a/go.sum b/go.sum
index 7da521271a..c176b49953 100644
--- a/go.sum
+++ b/go.sum
@@ -235,8 +235,8 @@ github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDk
github.com/cloudflare/cfssl v1.6.4 h1:NMOvfrEjFfC63K3SGXgAnFdsgkmiq4kATme5BfcqrO8=
github.com/cloudflare/cfssl v1.6.4/go.mod h1:8b3CQMxfWPAeom3zBnGJ6sd+G1NkL5TXqmDXacb+1J0=
github.com/cloudflare/circl v1.1.0/go.mod h1:prBCrKB9DV4poKZY1l9zBXg2QJY7mvgRvtMxxK7fi4I=
-github.com/cloudflare/circl v1.3.7 h1:qlCDlTPz2n9fu58M0Nh1J/JzcFpfgkFHHX3O35r5vcU=
-github.com/cloudflare/circl v1.3.7/go.mod h1:sRTcRWXGLrKw6yIGJ+l7amYJFfAXbZG0kBSc8r4zxgA=
+github.com/cloudflare/circl v1.3.8 h1:j+V8jJt09PoeMFIu2uh5JUyEaIHTXVOHslFoLNAKqwI=
+github.com/cloudflare/circl v1.3.8/go.mod h1:PDRU+oXvdD7KCtgKxW95M5Z8BpSCJXQORiZFnBQS5QU=
github.com/cncf/udpa/go v0.0.0-20191209042840-269d4d468f6f/go.mod h1:M8M6+tZqaGXZJjfX53e64911xZQV5JYwmTeXPW+k8Sc=
github.com/cnf/structhash v0.0.0-20201127153200-e1b16c1ebc08 h1:ox2F0PSMlrAAiAdknSRMDrAr8mfxPCfSZolH+/qQnyQ=
github.com/cnf/structhash v0.0.0-20201127153200-e1b16c1ebc08/go.mod h1:pCxVEbcm3AMg7ejXyorUXi6HQCzOIBf7zEDVPtw0/U4=
@@ -367,8 +367,6 @@ github.com/go-logfmt/logfmt v0.3.0/go.mod h1:Qt1PoO58o5twSAckw1HlFXLmHsOX5/0LbT9
github.com/go-logfmt/logfmt v0.4.0/go.mod h1:3RMwSq7FuexP4Kalkev3ejPJsZTpXXBr9+V4qmtdjCk=
github.com/go-logfmt/logfmt v0.5.0/go.mod h1:wCYkCAKZfumFQihp8CzCvQ3paCTfi41vtzG1KdI/P7A=
github.com/go-logfmt/logfmt v0.5.1/go.mod h1:WYhtIu8zTZfxdn5+rREduYbwxfcBr/Vr6KEVveWlfTs=
-github.com/go-logr/logr v1.2.4 h1:g01GSCwiDw2xSZfjJ2/T9M+S6pFdcNtFYsp+Y43HYDQ=
-github.com/go-logr/logr v1.2.4/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A=
github.com/go-ole/go-ole v1.2.6 h1:/Fpf6oFPoeFik9ty7siob0G6Ke8QvQEuVcuChpwXzpY=
github.com/go-ole/go-ole v1.2.6/go.mod h1:pprOEPIfldk/42T2oK7lQ4v4JSDwmV0As9GaiUsvbm0=
github.com/go-openapi/jsonpointer v0.20.2 h1:mQc3nmndL8ZBzStEo3JYF8wzmeWffDH4VbXz58sAx6Q=
@@ -394,8 +392,6 @@ github.com/go-sql-driver/mysql v1.4.0/go.mod h1:zAC/RDZ24gD3HViQzih4MyKcchzm+sOG
github.com/go-sql-driver/mysql v1.7.1 h1:lUIinVbN1DY0xBg0eMOzmmtGoHwWBbvnWubQUrtU8EI=
github.com/go-sql-driver/mysql v1.7.1/go.mod h1:OXbVy3sEdcQ2Doequ6Z5BW6fXNQTmx+9S1MCJN5yJMI=
github.com/go-stack/stack v1.8.0/go.mod h1:v0f6uXyyMGvRgIKkXu+yp6POWl0qKG85gN/melR3HDY=
-github.com/go-task/slim-sprig v0.0.0-20230315185526-52ccab3ef572 h1:tfuBGBXKqDEevZMzYi5KSi8KkcZtzBcTgAUUtapy0OI=
-github.com/go-task/slim-sprig v0.0.0-20230315185526-52ccab3ef572/go.mod h1:9Pwr4B2jHnOSGXyyzV8ROjYa2ojvAY6HCGYYfMoC3Ls=
github.com/go-test/deep v1.0.8 h1:TDsG77qcSprGbC6vTN8OuXp5g+J+b5Pcguhf7Zt61VM=
github.com/go-test/deep v1.0.8/go.mod h1:5C2ZWiW0ErCdrYzpqxLbTX7MG14M9iiw8DgHncVwcsE=
github.com/goburrow/cache v0.1.4 h1:As4KzO3hgmzPlnaMniZU9+VmoNYseUhuELbxy9mRBfw=
@@ -630,8 +626,8 @@ github.com/kisielk/errcheck v1.5.0/go.mod h1:pFxgyoBC7bSaBwPgfKdkLd5X25qrDl4LWUI
github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck=
github.com/klauspost/compress v1.4.1/go.mod h1:RyIbtBH6LamlWaDj8nUwkbUhJ87Yi3uG0guNDohfE1A=
github.com/klauspost/compress v1.11.4/go.mod h1:aoV0uJVorq1K+umq18yTdKaF57EivdYsUV+/s2qKfXs=
-github.com/klauspost/compress v1.17.6 h1:60eq2E/jlfwQXtvZEeBUYADs+BwKBWURIY+Gj2eRGjI=
-github.com/klauspost/compress v1.17.6/go.mod h1:/dCuZOvVtNoHsyb+cuJD3itjs3NbnF6KH9zAO4BDxPM=
+github.com/klauspost/compress v1.17.8 h1:YcnTYrq7MikUT7k0Yb5eceMmALQPYBW/Xltxn0NAMnU=
+github.com/klauspost/compress v1.17.8/go.mod h1:Di0epgTjJY877eYKx5yC51cX2A2Vl2ibi7bDH9ttBbw=
github.com/klauspost/cpuid v1.2.0/go.mod h1:Pj4uuM528wm8OyEC2QMXAi2YiTZ96dNQPGgoMS4s3ek=
github.com/klauspost/cpuid/v2 v2.0.9/go.mod h1:FInQzS24/EEf25PyTYn52gqo7WaD8xa0213Md/qVLRg=
github.com/klauspost/cpuid/v2 v2.0.12/go.mod h1:g2LTdtYhdyuGPqyWyv7qRAmj1WBqxuObKfj5c0PQa7c=
@@ -662,8 +658,8 @@ github.com/leodido/go-urn v1.2.4 h1:XlAE/cm/ms7TE/VMVoduSpNBoyc2dOxHs5MZSwAN63Q=
github.com/leodido/go-urn v1.2.4/go.mod h1:7ZrI8mTSeBSHl/UaRyKQW1qZeMgak41ANeCNaVckg+4=
github.com/leslie-qiwa/flat v0.0.0-20230424180412-f9d1cf014baa h1:KQKuQDgA3DZX6C396lt3WDYB9Um1gLITLbvficVbqXk=
github.com/leslie-qiwa/flat v0.0.0-20230424180412-f9d1cf014baa/go.mod h1:HbwNE4XGwjgtUELkvQaAOjWrpianHYZdQVNqSdYW3UM=
-github.com/lib/pq v1.10.1 h1:6VXZrLU0jHBYyAqrSPa+MgPfnSvTPuMgK+k0o5kVFWo=
-github.com/lib/pq v1.10.1/go.mod h1:AlVN5x4E4T544tWzH6hKfbfQvm3HdbOxrmggDNAPY9o=
+github.com/lib/pq v1.10.9 h1:YXG7RB+JIjhP29X+OtkiDnYaXQwpS4JEWq7dtCCRUEw=
+github.com/lib/pq v1.10.9/go.mod h1:AlVN5x4E4T544tWzH6hKfbfQvm3HdbOxrmggDNAPY9o=
github.com/libdns/libdns v0.2.1 h1:Wu59T7wSHRgtA0cfxC+n1c/e+O3upJGWytknkmFEDis=
github.com/libdns/libdns v0.2.1/go.mod h1:yQCXzk1lEZmmCPa857bnk4TsOiqYasqpyOEeSObbb40=
github.com/lightstep/lightstep-tracer-common/golang/gogo v0.0.0-20190605223551-bc2310a04743/go.mod h1:qklhhLq1aX+mtWk9cPHPzaBjWImj5ULL6C7HFJtXQMM=
@@ -714,8 +710,8 @@ github.com/microcosm-cc/bluemonday v1.0.26 h1:xbqSvqzQMeEHCqMi64VAs4d8uy6Mequs3r
github.com/microcosm-cc/bluemonday v1.0.26/go.mod h1:JyzOCs9gkyQyjs+6h10UEVSe02CGwkhd72Xdqh78TWs=
github.com/miekg/dns v1.0.14/go.mod h1:W1PPwlIAgtquWBMBEV9nkV9Cazfe8ScdGz/Lj7v3Nrg=
github.com/miekg/dns v1.1.35/go.mod h1:KNUDUusw/aVsxyTYZM1oqvCicbwhgbNgztCETuNZ7xM=
-github.com/miekg/dns v1.1.57 h1:Jzi7ApEIzwEPLHWRcafCN9LZSBbqQpxjt/wpgvg7wcM=
-github.com/miekg/dns v1.1.57/go.mod h1:uqRjCRUuEAA6qsOiJvDd+CFo/vW+y5WR6SNmHE55hZk=
+github.com/miekg/dns v1.1.59 h1:C9EXc/UToRwKLhK5wKU/I4QVsBUc8kE6MkHBkeypWZs=
+github.com/miekg/dns v1.1.59/go.mod h1:nZpewl5p6IvctfgrckopVx2OlSEHPRO/U4SYkRklrEk=
github.com/minio/minio-go/v6 v6.0.46/go.mod h1:qD0lajrGW49lKZLtXKtCB4X/qkMf0a5tBvN2PaZg7Gg=
github.com/minio/selfupdate v0.6.1-0.20230907112617-f11e74f84ca7 h1:yRZGarbxsRytL6EGgbqK2mCY+Lk5MWKQYKJT2gEglhc=
github.com/minio/selfupdate v0.6.1-0.20230907112617-f11e74f84ca7/go.mod h1:bO02GTIPCMQFTEvE5h4DjYB58bCoZ35XLeBf0buTDdM=
@@ -773,8 +769,6 @@ github.com/onsi/ginkgo v1.6.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+W
github.com/onsi/ginkgo v1.7.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE=
github.com/onsi/ginkgo v1.16.4 h1:29JGrr5oVBm5ulCWet69zQkzWipVXIol6ygQUe/EzNc=
github.com/onsi/ginkgo v1.16.4/go.mod h1:dX+/inL/fNMqNlz0e9LfyB9TswhZpCVdJM/Z6Vvnwo0=
-github.com/onsi/ginkgo/v2 v2.9.5 h1:+6Hr4uxzP4XIUyAkg61dWBw8lb/gc4/X5luuxN/EC+Q=
-github.com/onsi/ginkgo/v2 v2.9.5/go.mod h1:tvAoo1QUJwNEU2ITftXTpR7R1RbCzoZUOs3RonqW57k=
github.com/onsi/gomega v1.4.3/go.mod h1:ex+gbHU/CVuBBDIJjb2X0qEXbFg53c61hWP/1CpauHY=
github.com/onsi/gomega v1.27.10 h1:naR28SdDFlqrG6kScpT8VWpu1xWY5nJRCF3XaYyBjhI=
github.com/onsi/gomega v1.27.10/go.mod h1:RsS8tutOdbdgzbPtzzATp12yT7kM5I5aElG3evPbQ0M=
@@ -837,8 +831,8 @@ github.com/projectdiscovery/clistats v0.0.20 h1:5jO5SLiRJ7f0nDV0ndBNmBeesbROouPo
github.com/projectdiscovery/clistats v0.0.20/go.mod h1:GJ2av0KnOvK0AISQnP8hyDclYIji1LVkx2l0pwnzAu4=
github.com/projectdiscovery/dsl v0.0.56 h1:iVFIfDdGXkrXTh5sf6hRaxqTTEYiv/LnNjoOHJIfHiY=
github.com/projectdiscovery/dsl v0.0.56/go.mod h1:3UBjPvtiy8c5E8oAUrKobMKjb3kEwQuGB5tqIN964Og=
-github.com/projectdiscovery/fastdialer v0.0.72 h1:CbKNFqvJotGmn9uBeHoR2vJQRg8QMuQs9NIOc8HW02E=
-github.com/projectdiscovery/fastdialer v0.0.72/go.mod h1:sfeBKELnLnkpwEYcK5Qf8DRXLcdmR34u4TxtFwxNNQ0=
+github.com/projectdiscovery/fastdialer v0.1.0 h1:NiId7eXvYZjiBo83AnIZ6K8ZjzLJU8wGXdRLzuKcRrg=
+github.com/projectdiscovery/fastdialer v0.1.0/go.mod h1:jKOWzyHx+Q2sMiYst5HP6tWLQLDFPVgWBR72szn2h8w=
github.com/projectdiscovery/fasttemplate v0.0.2 h1:h2cISk5xDhlJEinlBQS6RRx0vOlOirB2y3Yu4PJzpiA=
github.com/projectdiscovery/fasttemplate v0.0.2/go.mod h1:XYWWVMxnItd+r0GbjA1GCsUopMw1/XusuQxdyAIHMCw=
github.com/projectdiscovery/freeport v0.0.5 h1:jnd3Oqsl4S8n0KuFkE5Hm8WGDP24ITBvmyw5pFTHS8Q=
@@ -853,12 +847,12 @@ github.com/projectdiscovery/gostruct v0.0.2 h1:s8gP8ApugGM4go1pA+sVlPDXaWqNP5BBD
github.com/projectdiscovery/gostruct v0.0.2/go.mod h1:H86peL4HKwMXcQQtEa6lmC8FuD9XFt6gkNR0B/Mu5PE=
github.com/projectdiscovery/gozero v0.0.2 h1:8fJeaCjxL9tpm33uG/RsCQs6HGM/NE6eA3cjkilRQ+E=
github.com/projectdiscovery/gozero v0.0.2/go.mod h1:d8bZvDWW07LWNYWrwjZ4OO1I0cpkfqaysyDfSs9ibK8=
-github.com/projectdiscovery/hmap v0.0.42 h1:+P8CC7gAeTG0phe0d1FB7i3Vl15v1K+dJApwX4rvMAM=
-github.com/projectdiscovery/hmap v0.0.42/go.mod h1:lbGBuL/bLoYWdlgphZmHXjZCYzteVDf4WfKsR/aH57c=
+github.com/projectdiscovery/hmap v0.0.43 h1:9A/lGm/0uP9x2hP90dmK5vkc6KfGD0XYxDBAQ7lx/zg=
+github.com/projectdiscovery/hmap v0.0.43/go.mod h1:cMLXGlkLcPK0Y/aJVa+IZM9wdtkQ6HX4mHTXVC05Jbc=
github.com/projectdiscovery/httpx v1.6.1 h1:aqKzoSydzKAJBV3Sxbqt44nTah4fNS9GkSD4Dy6Fzlc=
github.com/projectdiscovery/httpx v1.6.1/go.mod h1:RvibGQbm2Bh2DtuBDVXHLfauFDLtp7opKHyk65sz0ss=
-github.com/projectdiscovery/interactsh v1.1.10-0.20240519152711-6a0cb98b1663 h1:0ih5JJyTht2sv4J4lbfSPaUkKdFDci+Bwxg6vsyd2F4=
-github.com/projectdiscovery/interactsh v1.1.10-0.20240519152711-6a0cb98b1663/go.mod h1:gTrQCQfYlhoIM4X11+zwL7uRuraOUCoiE4Z6z1QD86g=
+github.com/projectdiscovery/interactsh v1.1.9 h1:b77SaSGrO+DtivmWwqGGY2dmNlQC3Zgmwlaj9L4Oqvc=
+github.com/projectdiscovery/interactsh v1.1.9/go.mod h1:0FRQXCildcTLq7Tsa4BVZAsFCXhpWs4C4quKWigXb5I=
github.com/projectdiscovery/ldapserver v1.0.2-0.20240219154113-dcc758ebc0cb h1:MGtI4oE12ruWv11ZlPXXd7hl/uAaQZrFvrIDYDeVMd8=
github.com/projectdiscovery/ldapserver v1.0.2-0.20240219154113-dcc758ebc0cb/go.mod h1:vmgC0DTFCfoCLp0RAfsfYTZZan0QMVs+cmTbH6blfjk=
github.com/projectdiscovery/machineid v0.0.0-20240226150047-2e2c51e35983 h1:ZScLodGSezQVwsQDtBSMFp72WDq0nNN+KE/5DHKY5QE=
@@ -871,14 +865,14 @@ github.com/projectdiscovery/networkpolicy v0.0.8 h1:XvfBaBwSDNTesSfNQP9VLk3HX9I7
github.com/projectdiscovery/networkpolicy v0.0.8/go.mod h1:xnjNqhemxUPxU+UD5Jgsc3+K8IVmcqT1SJeo6UzMtkI=
github.com/projectdiscovery/ratelimit v0.0.42 h1:110tBLaGUgyPK0DjLmMBQaaqW4tkmqsFidr/t6tyOy4=
github.com/projectdiscovery/ratelimit v0.0.42/go.mod h1:ruhLiZ5liukSpG07p6eHTCPJUmTwOhxDSxQPulvC3/Y=
-github.com/projectdiscovery/rawhttp v0.1.50 h1:JQw0jBnUEcU2ZoTi8TBQpxHKykxkoBSfh9WETwggZIM=
-github.com/projectdiscovery/rawhttp v0.1.50/go.mod h1:h3PiVqB8w7U/yOK+TCAJb1/zu0FHPCFM2ZQ1k2qm9cg=
+github.com/projectdiscovery/rawhttp v0.1.49 h1:OPP9R/UZx/GFrcPRUs9fOS1dLwhg+2o7p3dByzkIhWM=
+github.com/projectdiscovery/rawhttp v0.1.49/go.mod h1:aaAaMsdzHPfw4yU3nbeP7NI3vy1ZjgoXw7l+m4Tnt94=
github.com/projectdiscovery/rdap v0.9.1-0.20221108103045-9865884d1917 h1:m03X4gBVSorSzvmm0bFa7gDV4QNSOWPL/fgZ4kTXBxk=
github.com/projectdiscovery/rdap v0.9.1-0.20221108103045-9865884d1917/go.mod h1:JxXtZC9e195awe7EynrcnBJmFoad/BNDzW9mzFkK8Sg=
-github.com/projectdiscovery/retryabledns v1.0.59 h1:8pMN+VibEBp29RIUior9LXUbx0RsBTjPC0008t2hfGU=
-github.com/projectdiscovery/retryabledns v1.0.59/go.mod h1:CwyQLDt9oqNIO/2ArALhAnUHJjZYdvJRSfGERRNPtoQ=
-github.com/projectdiscovery/retryablehttp-go v1.0.60 h1:sXbx6Rdh22SZ3AFhY3P7LC+p8GPLlANMgPHlkBXJlv8=
-github.com/projectdiscovery/retryablehttp-go v1.0.60/go.mod h1:rgRdV7LSrrTTlvN7yKsYxtvWm39VZB6pgD2t1p1ma64=
+github.com/projectdiscovery/retryabledns v1.0.60 h1:f3TPeLnaJKrl9CbfsTfFTFnJY+UnyArH3mxxUUyR5ZQ=
+github.com/projectdiscovery/retryabledns v1.0.60/go.mod h1:T4Su40Wa9lVtRNMfMDFJi00g2T3FbTfwnKKkYON0WgU=
+github.com/projectdiscovery/retryablehttp-go v1.0.62 h1:ZY09NYRatoELHaPUcdRdWzHIJExjEqceHAfe7u/e9ZY=
+github.com/projectdiscovery/retryablehttp-go v1.0.62/go.mod h1:yhRy9Q3z0CuNUHIIPrveaubjVEwF4arKWT/5SQurQEE=
github.com/projectdiscovery/sarif v0.0.1 h1:C2Tyj0SGOKbCLgHrx83vaE6YkzXEVrMXYRGLkKCr/us=
github.com/projectdiscovery/sarif v0.0.1/go.mod h1:cEYlDu8amcPf6b9dSakcz2nNnJsoz4aR6peERwV+wuQ=
github.com/projectdiscovery/stringsutil v0.0.2 h1:uzmw3IVLJSMW1kEg8eCStG/cGbYYZAja8BH3LqqJXMA=
@@ -889,8 +883,8 @@ github.com/projectdiscovery/uncover v1.0.7 h1:ut+2lTuvmftmveqF5RTjMWAgyLj8ltPQC7
github.com/projectdiscovery/uncover v1.0.7/go.mod h1:HFXgm1sRPuoN0D4oATljPIdmbo/EEh1wVuxQqo/dwFE=
github.com/projectdiscovery/useragent v0.0.52 h1:9SUPH0Epo3DJfB6PCDgETfMaD6nZ08sFvfgXTmPUAsU=
github.com/projectdiscovery/useragent v0.0.52/go.mod h1:PUXHgShvaD8p3bihy1mY8tuBDhdk3M0yy4Z10Ajg2yQ=
-github.com/projectdiscovery/utils v0.0.94 h1:2zzFEjMkq/Ei/o3NIA2SWTkhfGHMkBy0T3aIzq0vizo=
-github.com/projectdiscovery/utils v0.0.94/go.mod h1:wxPi+kCsLm5JCLMkZJyGwS+4Mn4PaPHHf0ayE8JphOw=
+github.com/projectdiscovery/utils v0.1.0 h1:r7Z/s2CBktJ0bnSN410lzOhD8S/0IxmzmFxkQudYKps=
+github.com/projectdiscovery/utils v0.1.0/go.mod h1:RaBdJLTKF5FKZ/RtMeccqFBtpsSjaggVw6/oPTpDD40=
github.com/projectdiscovery/wappalyzergo v0.1.0 h1:ZagOIKemBsNfCDRQeWavWEXtEjP8UiziuoRoPTwDAJ0=
github.com/projectdiscovery/wappalyzergo v0.1.0/go.mod h1:wBYGKmA5BQp/NWsAy1q/jSH8N1LHWQ/LV26DuR+KzPM=
github.com/projectdiscovery/yamldoc-go v1.0.4 h1:eZoESapnMw6WAHiVgRwNqvbJEfNHEH148uthhFbG5jE=
@@ -927,13 +921,11 @@ github.com/prometheus/procfs v0.6.0/go.mod h1:cz+aTbrPOrUb4q7XlbU9ygM+/jj0fzG6c1
github.com/prometheus/procfs v0.7.3/go.mod h1:cz+aTbrPOrUb4q7XlbU9ygM+/jj0fzG6c1xBZuNvfVA=
github.com/prometheus/procfs v0.8.0/go.mod h1:z7EfXMXOkbkqb9IINtpCn86r/to3BnA0uaxHdg830/4=
github.com/prometheus/procfs v0.9.0/go.mod h1:+pB4zwohETzFnmlpe6yd2lSc+0/46IYZRB/chUwxUZY=
-github.com/quic-go/quic-go v0.42.0 h1:uSfdap0eveIl8KXnipv9K7nlwZ5IqLlYOpJ58u5utpM=
-github.com/quic-go/quic-go v0.42.0/go.mod h1:132kz4kL3F9vxhW3CtQJLDVwcFe5wdWeJXXijhsO57M=
github.com/rcrowley/go-metrics v0.0.0-20181016184325-3113b8401b8a/go.mod h1:bCqnVzQkZxMG4s8nGwiZ5l3QUCyqpo9Y+/ZMZ9VjZe4=
github.com/redis/go-redis/v9 v9.1.0 h1:137FnGdk+EQdCbye1FW+qOEcY5S+SpY9T0NiuqvtfMY=
github.com/redis/go-redis/v9 v9.1.0/go.mod h1:urWj3He21Dj5k4TK1y59xH8Uj6ATueP8AH1cY3lZl4c=
-github.com/refraction-networking/utls v1.6.1 h1:n1JG5karzdGWsI6iZmGrOv3SNzR4c+4M8J6KWGsk3lA=
-github.com/refraction-networking/utls v1.6.1/go.mod h1:+EbcQOvQvXoFV9AEKbuGlljt1doLRKAVY1jJHe9EtDo=
+github.com/refraction-networking/utls v1.6.6 h1:igFsYBUJPYM8Rno9xUuDoM5GQrVEqY4llzEXOkL43Ig=
+github.com/refraction-networking/utls v1.6.6/go.mod h1:BC3O4vQzye5hqpmDTWUqi4P5DDhzJfkV1tdqtawQIH0=
github.com/remeh/sizedwaitgroup v1.0.0 h1:VNGGFwNo/R5+MJBf6yrsr110p0m4/OX4S3DCy7Kyl5E=
github.com/remeh/sizedwaitgroup v1.0.0/go.mod h1:3j2R4OIe/SeS6YDhICBy22RWjJC5eNCJ1V+9+NVNYlo=
github.com/rivo/uniseg v0.1.0/go.mod h1:J6wj4VEh+S6ZtnVlnTBMWIodfgj8LQOQFoIToxlJtxc=
@@ -1015,17 +1007,17 @@ github.com/stretchr/testify v1.9.0 h1:HtqpIVDClZ4nwg75+f6Lvsy/wHu+3BoSGCbBAcpTsT
github.com/stretchr/testify v1.9.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY=
github.com/syndtr/goleveldb v1.0.0 h1:fBdIW9lB4Iz0n9khmH8w27SJ3QEJ7+IgjPEwGSZiFdE=
github.com/syndtr/goleveldb v1.0.0/go.mod h1:ZVVdQEZoIme9iO1Ch2Jdy24qqXrMMOU6lpPAyBWyWuQ=
-github.com/tarunKoyalwar/goleak v0.0.0-20240426214851-746d64600adc h1:/5P5I7oDqdLee8W9Moof0xSD8tT1qEVzhObSI9CqHkg=
-github.com/tarunKoyalwar/goleak v0.0.0-20240426214851-746d64600adc/go.mod h1:uQdBQGrE1fZ2EyOs0pLcCDd1bBV4rSThieuIIGhXZ50=
+github.com/tarunKoyalwar/goleak v0.0.0-20240429141123-0efa90dbdcf9 h1:GXIyLuIJ5Qk46lI8WJ83qHBZKUI3zhmMmuoY9HICUIQ=
+github.com/tarunKoyalwar/goleak v0.0.0-20240429141123-0efa90dbdcf9/go.mod h1:uQdBQGrE1fZ2EyOs0pLcCDd1bBV4rSThieuIIGhXZ50=
github.com/tidwall/assert v0.1.0 h1:aWcKyRBUAdLoVebxo95N7+YZVTFF/ASTr7BN4sLP6XI=
github.com/tidwall/assert v0.1.0/go.mod h1:QLYtGyeqse53vuELQheYl9dngGCJQ+mTtlxcktb+Kj8=
github.com/tidwall/btree v1.7.0 h1:L1fkJH/AuEh5zBnnBbmTwQ5Lt+bRJ5A8EWecslvo9iI=
github.com/tidwall/btree v1.7.0/go.mod h1:twD9XRA5jj9VUQGELzDO4HPQTNJsoWWfYEL+EUQ2cKY=
-github.com/tidwall/buntdb v1.3.0 h1:gdhWO+/YwoB2qZMeAU9JcWWsHSYU3OvcieYgFRS0zwA=
-github.com/tidwall/buntdb v1.3.0/go.mod h1:lZZrZUWzlyDJKlLQ6DKAy53LnG7m5kHyrEHvvcDmBpU=
+github.com/tidwall/buntdb v1.3.1 h1:HKoDF01/aBhl9RjYtbaLnvX9/OuenwvQiC3OP1CcL4o=
+github.com/tidwall/buntdb v1.3.1/go.mod h1:lZZrZUWzlyDJKlLQ6DKAy53LnG7m5kHyrEHvvcDmBpU=
github.com/tidwall/gjson v1.12.1/go.mod h1:/wbyibRr2FHMks5tjHJ5F8dMZh3AcwJEMf5vlfC0lxk=
-github.com/tidwall/gjson v1.17.0 h1:/Jocvlh98kcTfpN2+JzGQWQcqrPQwDrVEMApx/M5ZwM=
-github.com/tidwall/gjson v1.17.0/go.mod h1:/wbyibRr2FHMks5tjHJ5F8dMZh3AcwJEMf5vlfC0lxk=
+github.com/tidwall/gjson v1.17.1 h1:wlYEnwqAHgzmhNUFfw7Xalt2JzQvsMx2Se4PcoFCT/U=
+github.com/tidwall/gjson v1.17.1/go.mod h1:/wbyibRr2FHMks5tjHJ5F8dMZh3AcwJEMf5vlfC0lxk=
github.com/tidwall/grect v0.1.4 h1:dA3oIgNgWdSspFzn1kS4S/RDpZFLrIxAZOdJKjYapOg=
github.com/tidwall/grect v0.1.4/go.mod h1:9FBsaYRaR0Tcy4UwefBX/UDcDcDy9V5jUcxHzv2jd5Q=
github.com/tidwall/lotsa v1.0.2 h1:dNVBH5MErdaQ/xd9s769R31/n2dXavsQ0Yf4TMEHHw8=
@@ -1068,8 +1060,8 @@ github.com/valyala/fasttemplate v1.2.2/go.mod h1:KHLXt3tVN2HBp8eijSv/kGJopbvo7S+
github.com/weppos/publicsuffix-go v0.12.0/go.mod h1:z3LCPQ38eedDQSwmsSRW4Y7t2L8Ln16JPQ02lHAdn5k=
github.com/weppos/publicsuffix-go v0.13.0/go.mod h1:z3LCPQ38eedDQSwmsSRW4Y7t2L8Ln16JPQ02lHAdn5k=
github.com/weppos/publicsuffix-go v0.30.0/go.mod h1:kBi8zwYnR0zrbm8RcuN1o9Fzgpnnn+btVN8uWPMyXAY=
-github.com/weppos/publicsuffix-go v0.30.2-0.20230730094716-a20f9abcc222 h1:h2JizvZl9aIj6za9S5AyrkU+OzIS4CetQthH/ejO+lg=
-github.com/weppos/publicsuffix-go v0.30.2-0.20230730094716-a20f9abcc222/go.mod h1:s41lQh6dIsDWIC1OWh7ChWJXLH0zkJ9KHZVqA7vHyuQ=
+github.com/weppos/publicsuffix-go v0.30.2 h1:Np18yzfMR90jNampWFs7iSh2sw/qCZkhL41/ffyihCU=
+github.com/weppos/publicsuffix-go v0.30.2/go.mod h1:/hGscit36Yt+wammfBBwdMdxBT8btsTt6KvwO9OvMyM=
github.com/weppos/publicsuffix-go/publicsuffix/generator v0.0.0-20220927085643-dc0d00c92642/go.mod h1:GHfoeIdZLdZmLjMlzBftbTDntahTttUMWjxZwQJhULE=
github.com/wk8/go-ordered-map/v2 v2.1.8 h1:5h/BUHu93oj4gIdvHHHGsScSTMijfx5PeYkE/fJgbpc=
github.com/wk8/go-ordered-map/v2 v2.1.8/go.mod h1:5nJHM5DyteebpVlHnWMV0rPz6Zp7+xBAnxjb1X5vnTw=
@@ -1132,15 +1124,15 @@ github.com/zmap/zcertificate v0.0.1/go.mod h1:q0dlN54Jm4NVSSuzisusQY0hqDWvu92C+T
github.com/zmap/zcrypto v0.0.0-20201128221613-3719af1573cf/go.mod h1:aPM7r+JOkfL+9qSB4KbYjtoEzJqUK50EXkkJabeNJDQ=
github.com/zmap/zcrypto v0.0.0-20201211161100-e54a5822fb7e/go.mod h1:aPM7r+JOkfL+9qSB4KbYjtoEzJqUK50EXkkJabeNJDQ=
github.com/zmap/zcrypto v0.0.0-20230310154051-c8b263fd8300/go.mod h1:mOd4yUMgn2fe2nV9KXsa9AyQBFZGzygVPovsZR+Rl5w=
-github.com/zmap/zcrypto v0.0.0-20231219022726-a1f61fb1661c h1:U1b4THKcgOpJ+kILupuznNwPiURtwVW3e9alJvji9+s=
-github.com/zmap/zcrypto v0.0.0-20231219022726-a1f61fb1661c/go.mod h1:GSDpFDD4TASObxvfZfvpZZ3OWHIUHMlhVWlkOe4ewVk=
+github.com/zmap/zcrypto v0.0.0-20240512203510-0fef58d9a9db h1:IfONOhyZlf4qPt3ENPU+27mBbPjzTQ+swKpj7MJva9I=
+github.com/zmap/zcrypto v0.0.0-20240512203510-0fef58d9a9db/go.mod h1:mo/07mo6reDaiz6BzveCuYBWb1d+aX8Pf8Nh+Q57y2g=
github.com/zmap/zflags v1.4.0-beta.1.0.20200204220219-9d95409821b6/go.mod h1:HXDUD+uue8yeLHr0eXx1lvY6CvMiHbTKw5nGmA9OUoo=
github.com/zmap/zgrab2 v0.1.8-0.20230806160807-97ba87c0e706 h1:LaMyYFWQA7kh3ovPfAaFDTKlJu3JGng8khruOtsBVnE=
github.com/zmap/zgrab2 v0.1.8-0.20230806160807-97ba87c0e706/go.mod h1:re2kMcs84XHb8Xl6RInt0emoKCuphfmfjHYuteviLHQ=
github.com/zmap/zlint/v3 v3.0.0/go.mod h1:paGwFySdHIBEMJ61YjoqT4h7Ge+fdYG4sUQhnTb1lJ8=
go.etcd.io/bbolt v1.3.3/go.mod h1:IbVyRI1SCnLcuJnV2u8VeU0CEYM7e686BmAb1XKL+uU=
-go.etcd.io/bbolt v1.3.8 h1:xs88BrvEv273UsB79e0hcVrlUWmS0a8upikMFhSyAtA=
-go.etcd.io/bbolt v1.3.8/go.mod h1:N9Mkw9X8x5fupy0IKsmuqVtoGDyxsaDlbk4Rd05IAQw=
+go.etcd.io/bbolt v1.3.10 h1:+BqfJTcCzTItrop8mq/lbzL8wSGtj94UO/3U31shqG0=
+go.etcd.io/bbolt v1.3.10/go.mod h1:bK3UQLPJZly7IlNmV7uVHJDxfe5aK9Ll93e/74Y9oEQ=
go.etcd.io/etcd v0.0.0-20191023171146-3cf2f69b5738/go.mod h1:dnLIgRNXwCJa5e+c6mIZCrds/GIG4ncV9HhK5PX7jPg=
go.opencensus.io v0.20.1/go.mod h1:6WKK9ahsWS3RSO+PY9ZHZUfv2irvY6gN279GOPZjmmk=
go.opencensus.io v0.20.2/go.mod h1:6WKK9ahsWS3RSO+PY9ZHZUfv2irvY6gN279GOPZjmmk=
@@ -1188,8 +1180,8 @@ golang.org/x/crypto v0.0.0-20220525230936-793ad666bf5e/go.mod h1:IxCIyHEi3zRg3s0
golang.org/x/crypto v0.0.0-20220622213112-05595931fe9d/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4=
golang.org/x/crypto v0.6.0/go.mod h1:OFC/31mSvZgRz0V1QTNCzfAI1aIRzbiufJtkMIlEp58=
golang.org/x/crypto v0.7.0/go.mod h1:pYwdfH91IfpZVANVyUOhSIPZaFoJGxTFbZhFTx+dXZU=
-golang.org/x/crypto v0.11.0/go.mod h1:xgJhtzW8F9jGdVFWZESrid1U1bjeNy4zgy5cRr/CIio=
-golang.org/x/crypto v0.17.0/go.mod h1:gCAAfMLgwOJRpTjQ2zCCt2OcSfYMTeZVSRtQlPC7Nq4=
+golang.org/x/crypto v0.14.0/go.mod h1:MVFd36DqK4CsrnJYDkBA3VC4m2GkXAM0PvzMCn4JQf4=
+golang.org/x/crypto v0.19.0/go.mod h1:Iy9bg/ha4yyC70EfRS8jz+B6ybOBKMaSxLj6P6oBDfU=
golang.org/x/crypto v0.23.0 h1:dIJU/v2J8Mdglj/8rJ6UUOM3Zc9zLZxVZwwxMooUSAI=
golang.org/x/crypto v0.23.0/go.mod h1:CKFgDieR+mRhux2Lsu27y0fO304Db0wZe70UKqHu0v8=
golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA=
@@ -1202,8 +1194,8 @@ golang.org/x/exp v0.0.0-20191227195350-da58074b4299/go.mod h1:2RIsYlXP63K8oxa1u0
golang.org/x/exp v0.0.0-20200119233911-0405dc783f0a/go.mod h1:2RIsYlXP63K8oxa1u096TMicItID8zy7Y6sNkU49FU4=
golang.org/x/exp v0.0.0-20200207192155-f17229e696bd/go.mod h1:J/WKrq2StrnmMY6+EHIKF9dgMWnmCNThgcyBT1FY9mM=
golang.org/x/exp v0.0.0-20200224162631-6cc2880d07d6/go.mod h1:3jZMyOhIsHpP37uCMkUooju7aAi5cS1Q23tOzKc+0MU=
-golang.org/x/exp v0.0.0-20240119083558-1b970713d09a h1:Q8/wZp0KX97QFTc2ywcOE0YRjZPVIx+MXInMzdvQqcA=
-golang.org/x/exp v0.0.0-20240119083558-1b970713d09a/go.mod h1:idGWGoKP1toJGkd5/ig9ZLuPcZBC3ewk7SzmH0uou08=
+golang.org/x/exp v0.0.0-20240506185415-9bf2ced13842 h1:vr/HnozRka3pE4EsMEg1lgkXJkTFJCVUX+S/ZT6wYzM=
+golang.org/x/exp v0.0.0-20240506185415-9bf2ced13842/go.mod h1:XtvwrStGgqGPLc4cjQfWqZHG1YFdYs6swckp8vpsjnc=
golang.org/x/image v0.0.0-20190227222117-0694c2d4d067/go.mod h1:kZ7UVZpmo3dzQBMxlp+ypCbDeSB+sBbTgSJuh5dn5js=
golang.org/x/image v0.0.0-20190802002840-cff245a6509b/go.mod h1:FeLwcggjj3mMvU+oOTbSwawSJRM1uh48EjtB4UJZlP0=
golang.org/x/lint v0.0.0-20181026193005-c67002cb31c3/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE=
@@ -1226,8 +1218,8 @@ golang.org/x/mod v0.2.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA=
golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA=
golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4/go.mod h1:jJ57K6gSWd91VN4djpZkiMVwK6gcyfeH4XE8wZrZaV4=
golang.org/x/mod v0.8.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs=
-golang.org/x/mod v0.14.0 h1:dGoOF9QVLYng8IHTm7BAyWqCqSheQ5pYWGhzW00YJr0=
-golang.org/x/mod v0.14.0/go.mod h1:hTbmBsO62+eylJbnUtE2MGJUyE7QWk4xUqPFrRgJ+7c=
+golang.org/x/mod v0.17.0 h1:zY54UmvipHiNd+pm+m0x9KhZ9hl1/7QNMyxXbc6ICqA=
+golang.org/x/mod v0.17.0/go.mod h1:hTbmBsO62+eylJbnUtE2MGJUyE7QWk4xUqPFrRgJ+7c=
golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
golang.org/x/net v0.0.0-20180906233101-161cd47e91fd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
@@ -1283,7 +1275,8 @@ golang.org/x/net v0.7.0/go.mod h1:2Tu9+aMcznHK/AK1HMvgo6xiTLG5rD5rZLDS+rp2Bjs=
golang.org/x/net v0.8.0/go.mod h1:QVkue5JL9kW//ek3r6jTKnTFis1tRmNAW2P1shuFdJc=
golang.org/x/net v0.9.0/go.mod h1:d48xBJpPfHeWQsugry2m+kC02ZBRGRgulfHnEXEuWns=
golang.org/x/net v0.10.0/go.mod h1:0qNGK6F8kojg2nk9dLZ2mShWaEBan6FAoqfSigmmuDg=
-golang.org/x/net v0.12.0/go.mod h1:zEVYFnQC7m/vmpQFELhcD1EWkZlX69l4oqgmer6hfKA=
+golang.org/x/net v0.17.0/go.mod h1:NxSsAGuq816PNPmqtQdLE42eU2Fs7NoRIZrHJAlaCOE=
+golang.org/x/net v0.21.0/go.mod h1:bIjVDfnllIU7BJ2DNgfnXvpSvtn8VRwhlsaeUTyUS44=
golang.org/x/net v0.25.0 h1:d/OCCoBEUq33pjydKrGQhw7IlUPI2Oylr+8qLx49kac=
golang.org/x/net v0.25.0/go.mod h1:JkAGAh7GEvH74S6FOH42FLoXpXbE/aqXSrIQjXgsiwM=
golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U=
@@ -1310,8 +1303,8 @@ golang.org/x/sync v0.0.0-20201207232520-09787c993a3a/go.mod h1:RxMgew5VJxzue5/jJ
golang.org/x/sync v0.0.0-20220601150217-0de741cfad7f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
golang.org/x/sync v0.1.0/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
-golang.org/x/sync v0.6.0 h1:5BMeUDZ7vkXGfEr1x9B4bRcTH4lpkTkpdh0T/J+qjbQ=
-golang.org/x/sync v0.6.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk=
+golang.org/x/sync v0.7.0 h1:YsImfSBoP9QPYL0xyKJPq0gcaJdG3rInoqxTWbfQu9M=
+golang.org/x/sync v0.7.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk=
golang.org/x/sys v0.0.0-20180823144017-11551d06cbcc/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
golang.org/x/sys v0.0.0-20180905080454-ebe1bf3edb33/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
@@ -1383,9 +1376,8 @@ golang.org/x/sys v0.5.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.7.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.8.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
-golang.org/x/sys v0.10.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.11.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
-golang.org/x/sys v0.15.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
+golang.org/x/sys v0.13.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.17.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
golang.org/x/sys v0.20.0 h1:Od9JTbYCk261bKm4M/mw7AklTlFYIa0bIp9BgSm1S8Y=
golang.org/x/sys v0.20.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
@@ -1398,8 +1390,8 @@ golang.org/x/term v0.5.0/go.mod h1:jMB1sMXY+tzblOD4FWmEbocvup2/aLOaQEp7JmGp78k=
golang.org/x/term v0.6.0/go.mod h1:m6U89DPEgQRMq3DNkDClhWw02AUbt2daBVO4cn4Hv9U=
golang.org/x/term v0.7.0/go.mod h1:P32HKFT3hSsZrRxla30E9HqToFYAQPCMs/zFMBUFqPY=
golang.org/x/term v0.8.0/go.mod h1:xPskH00ivmX89bAKVGSKKtLOWNx2+17Eiy94tnKShWo=
-golang.org/x/term v0.10.0/go.mod h1:lpqdcUyK/oCiQxvxVrppt5ggO2KCZ5QblwqPnfZ6d5o=
-golang.org/x/term v0.15.0/go.mod h1:BDl952bC7+uMoWR75FIrCDx79TPU9oHkTZ9yRbYOrX0=
+golang.org/x/term v0.13.0/go.mod h1:LTmsnFJwVN6bCy1rVCoS+qHT1HhALEFxKncY3WNNh4U=
+golang.org/x/term v0.17.0/go.mod h1:lLRBjIVuehSbZlaOtGMbcMncT+aqLLLmKrsjNrUguwk=
golang.org/x/term v0.20.0 h1:VnkxpohqXaOBYJtBmEppKUG6mXpi+4O6purfc2+sMhw=
golang.org/x/term v0.20.0/go.mod h1:8UkIAJTvZgivsXaD6/pH6U9ecQzZ45awqEOzuCvwpFY=
golang.org/x/text v0.0.0-20170915032832-14c0d48ead0c/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
@@ -1415,7 +1407,7 @@ golang.org/x/text v0.6.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8=
golang.org/x/text v0.7.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8=
golang.org/x/text v0.8.0/go.mod h1:e1OnstbJyHTd6l/uOt8jFFHp6TRDWZR/bV3emEE/zU8=
golang.org/x/text v0.9.0/go.mod h1:e1OnstbJyHTd6l/uOt8jFFHp6TRDWZR/bV3emEE/zU8=
-golang.org/x/text v0.11.0/go.mod h1:TvPlkZtksWOMsz7fbANvkp4WM8x/WCo/om8BMLbz+aE=
+golang.org/x/text v0.13.0/go.mod h1:TvPlkZtksWOMsz7fbANvkp4WM8x/WCo/om8BMLbz+aE=
golang.org/x/text v0.14.0/go.mod h1:18ZOQIKpY8NJVqYksKHtTdi31H5itFRjB5/qKTNYzSU=
golang.org/x/text v0.15.0 h1:h1V/4gjBv8v9cjcR6+AR5+/cIYK5N/WAgiv4xlsEtAk=
golang.org/x/text v0.15.0/go.mod h1:18ZOQIKpY8NJVqYksKHtTdi31H5itFRjB5/qKTNYzSU=
@@ -1476,8 +1468,8 @@ golang.org/x/tools v0.0.0-20200825202427-b303f430e36d/go.mod h1:njjCfa9FT2d7l9Bc
golang.org/x/tools v0.0.0-20210106214847-113979e3529a/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA=
golang.org/x/tools v0.1.12/go.mod h1:hNGJHUnrk76NpqgfD5Aqm5Crs+Hm0VOH/i9J2+nxYbc=
golang.org/x/tools v0.6.0/go.mod h1:Xwgl3UAJ/d3gWutnCtw505GrjyAbvKui8lOU390QaIU=
-golang.org/x/tools v0.17.0 h1:FvmRgNOcs3kOa+T20R1uhfP9F6HgG2mfxDv1vrx1Htc=
-golang.org/x/tools v0.17.0/go.mod h1:xsh6VxdV005rRVaS6SSAf9oiAqljS7UZUacMZ8Bnsps=
+golang.org/x/tools v0.21.0 h1:qc0xYgIbsSDt9EyWz05J5wfa7LOVW0YTLOXrqdLAWIw=
+golang.org/x/tools v0.21.0/go.mod h1:aiJjzUbINMkxbQROHiO6hDPo2LHcIPhhQsa9DLh0yGk=
golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
diff --git a/internal/runner/options.go b/internal/runner/options.go
index cad5202f8a..3d36f51123 100644
--- a/internal/runner/options.go
+++ b/internal/runner/options.go
@@ -19,6 +19,7 @@ import (
"github.com/projectdiscovery/gologger/levels"
"github.com/projectdiscovery/nuclei/v3/pkg/catalog/config"
"github.com/projectdiscovery/nuclei/v3/pkg/protocols/common/protocolinit"
+ "github.com/projectdiscovery/nuclei/v3/pkg/protocols/common/protocolstate"
"github.com/projectdiscovery/nuclei/v3/pkg/protocols/common/utils/vardump"
"github.com/projectdiscovery/nuclei/v3/pkg/protocols/headless/engine"
"github.com/projectdiscovery/nuclei/v3/pkg/reporting"
@@ -114,7 +115,12 @@ func ParseOptions(options *types.Options) {
// Load the resolvers if user asked for them
loadResolvers(options)
- err := protocolinit.Init(options)
+ err := protocolstate.Init(options)
+ if err != nil {
+ gologger.Fatal().Msgf("Could not initialize protocol state: %s\n", err)
+ }
+
+ err = protocolinit.Init(options)
if err != nil {
gologger.Fatal().Msgf("Could not initialize protocols: %s\n", err)
}
diff --git a/internal/runner/runner.go b/internal/runner/runner.go
index a0c46f6d03..24010d7537 100644
--- a/internal/runner/runner.go
+++ b/internal/runner/runner.go
@@ -359,6 +359,10 @@ func (r *Runner) runStandardEnumeration(executerOpts protocols.ExecutorOptions,
// Close releases all the resources and cleans up
func (r *Runner) Close() {
+ // dump hosterrors cache
+ if r.hostErrors != nil {
+ r.hostErrors.Close()
+ }
if r.output != nil {
r.output.Close()
}
@@ -464,7 +468,7 @@ func (r *Runner) RunEnumeration() error {
Parser: r.parser,
}
- if env.GetEnvOrDefault("NUCLEI_ARGS", "") == "req_url_pattern=true" {
+ if config.DefaultConfig.IsDebugArgEnabled(config.DebugExportURLPattern) {
// Go StdLib style experimental/debug feature switch
executorOpts.ExportReqURLPattern = true
}
diff --git a/lib/multi.go b/lib/multi.go
index 5b2c7d6776..7c713e8aa0 100644
--- a/lib/multi.go
+++ b/lib/multi.go
@@ -123,7 +123,6 @@ func (e *ThreadSafeNucleiEngine) ExecuteNucleiWithOptsCtx(ctx context.Context, t
return err
}
}
- defer tmpEngine.Close()
// create ephemeral nuclei objects/instances/types using base nuclei engine
unsafeOpts, err := createEphemeralObjects(e.eng, tmpEngine.opts)
if err != nil {
diff --git a/lib/sdk_private.go b/lib/sdk_private.go
index 13e8746510..cfe9c88a25 100644
--- a/lib/sdk_private.go
+++ b/lib/sdk_private.go
@@ -25,7 +25,6 @@ import (
"github.com/projectdiscovery/nuclei/v3/pkg/protocols/common/hosterrorscache"
"github.com/projectdiscovery/nuclei/v3/pkg/protocols/common/interactsh"
"github.com/projectdiscovery/nuclei/v3/pkg/protocols/common/protocolinit"
- "github.com/projectdiscovery/nuclei/v3/pkg/protocols/common/protocolstate"
"github.com/projectdiscovery/nuclei/v3/pkg/protocols/http/httpclientpool"
"github.com/projectdiscovery/nuclei/v3/pkg/reporting"
"github.com/projectdiscovery/nuclei/v3/pkg/templates"
@@ -119,7 +118,6 @@ func (e *NucleiEngine) init() error {
e.parser = templates.NewParser()
sharedInit.Do(func() {
- _ = protocolstate.Init(e.opts)
_ = protocolinit.Init(e.opts)
})
diff --git a/pkg/catalog/config/constants.go b/pkg/catalog/config/constants.go
index 736671127d..3918a0124a 100644
--- a/pkg/catalog/config/constants.go
+++ b/pkg/catalog/config/constants.go
@@ -31,7 +31,7 @@ const (
CLIConfigFileName = "config.yaml"
ReportingConfigFilename = "reporting-config.yaml"
// Version is the current version of nuclei
- Version = `v3.2.7`
+ Version = `v3.2.8-dev`
// Directory Names of custom templates
CustomS3TemplatesDirName = "s3"
CustomGitHubTemplatesDirName = "github"
@@ -68,3 +68,13 @@ func trimDevIfExists(version string) string {
}
return version
}
+
+// similar to go pattern of enabling debug related features
+// we add custom/extra switches for debugging purposes
+const (
+ // DebugArgHostErrorStats is used to print host error stats
+ // when it is closed
+ DebugArgHostErrorStats = "host-error-stats"
+ // DebugExportReqURLPattern is used to export request URL pattern
+ DebugExportURLPattern = "req-url-pattern"
+)
diff --git a/pkg/catalog/config/nucleiconfig.go b/pkg/catalog/config/nucleiconfig.go
index 619d7fd47a..062393e542 100644
--- a/pkg/catalog/config/nucleiconfig.go
+++ b/pkg/catalog/config/nucleiconfig.go
@@ -46,9 +46,10 @@ type Config struct {
LatestNucleiIgnoreHash string `json:"nuclei-latest-ignore-hash,omitempty"`
// internal / unexported fields
- disableUpdates bool `json:"-"` // disable updates both version check and template updates
- homeDir string `json:"-"` // User Home Directory
- configDir string `json:"-"` // Nuclei Global Config Directory
+ disableUpdates bool `json:"-"` // disable updates both version check and template updates
+ homeDir string `json:"-"` // User Home Directory
+ configDir string `json:"-"` // Nuclei Global Config Directory
+ debugArgs []string `json:"-"` // debug args
}
// IsCustomTemplate determines whether a given template is custom-built or part of the official Nuclei templates.
@@ -329,6 +330,46 @@ func (c *Config) copyIgnoreFile() {
}
}
+// IsDebugArgEnabled checks if debug arg is enabled
+// this could be a feature specific to debugging like PPROF or printing stats
+// of max host error etc
+func (c *Config) IsDebugArgEnabled(arg string) bool {
+ for _, v := range c.debugArgs {
+ if v == arg {
+ return true
+ }
+ }
+ return false
+}
+
+// parseDebugArgs from string
+func (c *Config) parseDebugArgs(data string) {
+ // use space as seperator instead of commas
+ tmp := strings.Fields(data)
+ for _, v := range tmp {
+ key := v
+ value := ""
+ // if it is key value pair then split it
+ if strings.Contains(v, "=") {
+ parts := strings.SplitN(v, "=", 2)
+ if len(parts) != 2 {
+ continue
+ }
+ key, value = strings.TrimSpace(parts[0]), strings.TrimSpace(parts[1])
+ }
+ if value == "false" || value == "0" {
+ // if false or disabled then skip
+ continue
+ }
+ switch key {
+ case DebugArgHostErrorStats:
+ c.debugArgs = append(c.debugArgs, DebugArgHostErrorStats)
+ case DebugExportURLPattern:
+ c.debugArgs = append(c.debugArgs, DebugExportURLPattern)
+ }
+ }
+}
+
func init() {
// first attempt to migrate all files from old config directory to new config directory
goflags.AttemptConfigMigration() // regardless how many times this is called it will only migrate once based on condition
@@ -377,6 +418,7 @@ func init() {
// and even if it is changed we don't follow it since it is not expected behavior
// If custom templates are in default locations only then they are loaded while running nuclei
DefaultConfig.SetTemplatesDir(DefaultConfig.TemplatesDirectory)
+ DefaultConfig.parseDebugArgs(env.GetEnvOrDefault("NUCLEI_ARGS", ""))
}
// Add Default Config adds default when .templates-config.json file is not present
diff --git a/pkg/core/executors.go b/pkg/core/executors.go
index 86302bb713..0e91c3883b 100644
--- a/pkg/core/executors.go
+++ b/pkg/core/executors.go
@@ -107,7 +107,7 @@ func (e *Engine) executeTemplateWithTargets(ctx context.Context, template *templ
currentInfo.Unlock()
// Skip if the host has had errors
- if e.executerOpts.HostErrorsCache != nil && e.executerOpts.HostErrorsCache.Check(scannedValue.ID()) {
+ if e.executerOpts.HostErrorsCache != nil && e.executerOpts.HostErrorsCache.Check(contextargs.NewWithMetaInput(ctx, scannedValue)) {
return true
}
diff --git a/pkg/core/workflow_execute.go b/pkg/core/workflow_execute.go
index d0d3ede004..028df4d01f 100644
--- a/pkg/core/workflow_execute.go
+++ b/pkg/core/workflow_execute.go
@@ -98,7 +98,7 @@ func (e *Engine) runWorkflowStep(template *workflows.WorkflowTemplate, ctx *scan
}
if err != nil {
if w.Options.HostErrorsCache != nil {
- w.Options.HostErrorsCache.MarkFailed(ctx.Input.MetaInput.ID(), err)
+ w.Options.HostErrorsCache.MarkFailed(ctx.Input, err)
}
if len(template.Executers) == 1 {
mainErr = err
diff --git a/pkg/js/compiler/compiler.go b/pkg/js/compiler/compiler.go
index 05eacda6c6..367bd013b2 100644
--- a/pkg/js/compiler/compiler.go
+++ b/pkg/js/compiler/compiler.go
@@ -10,9 +10,15 @@ import (
"github.com/projectdiscovery/nuclei/v3/pkg/protocols/common/generators"
contextutil "github.com/projectdiscovery/utils/context"
+ "github.com/projectdiscovery/utils/errkit"
stringsutil "github.com/projectdiscovery/utils/strings"
)
+var (
+ // ErrJSExecDeadline is the error returned when alloted time for script execution exceeds
+ ErrJSExecDeadline = errkit.New("js engine execution deadline exceeded").SetKind(errkit.ErrKindDeadline).Build()
+)
+
// Compiler provides a runtime to execute goja runtime
// based javascript scripts efficiently while also
// providing them access to custom modules defined in libs/.
@@ -107,7 +113,7 @@ func (c *Compiler) ExecuteWithOptions(program *goja.Program, args *ExecuteArgs,
}
// execute with context and timeout
- ctx, cancel := context.WithTimeout(opts.Context, time.Duration(opts.Timeout)*time.Second)
+ ctx, cancel := context.WithTimeoutCause(opts.Context, time.Duration(opts.Timeout)*time.Second, ErrJSExecDeadline)
defer cancel()
// execute the script
results, err := contextutil.ExecFuncWithTwoReturns(ctx, func() (val goja.Value, err error) {
@@ -119,6 +125,9 @@ func (c *Compiler) ExecuteWithOptions(program *goja.Program, args *ExecuteArgs,
return ExecuteProgram(program, args, opts)
})
if err != nil {
+ if val, ok := err.(*goja.Exception); ok {
+ err = val.Unwrap()
+ }
return nil, err
}
var res ExecuteResult
diff --git a/pkg/js/compiler/init.go b/pkg/js/compiler/init.go
index 8a6d4a04be..8a171dc3da 100644
--- a/pkg/js/compiler/init.go
+++ b/pkg/js/compiler/init.go
@@ -1,6 +1,8 @@
package compiler
-import "github.com/projectdiscovery/nuclei/v3/pkg/types"
+import (
+ "github.com/projectdiscovery/nuclei/v3/pkg/types"
+)
// jsprotocolInit
@@ -9,6 +11,7 @@ var (
JsProtocolTimeout = 10
PoolingJsVmConcurrency = 100
NonPoolingVMConcurrency = 20
+ JsTimeoutMultiplier = 1.5
)
// Init initializes the javascript protocol
@@ -21,7 +24,9 @@ func Init(opts *types.Options) error {
// 100 is reasonable default
opts.JsConcurrency = 100
}
- JsProtocolTimeout = opts.Timeout
+ // we have dialer timeout set to 10s so js needs to be at least
+ // 15s to return the actual error if not it will be a dialer timeout
+ JsProtocolTimeout = int(float64(opts.Timeout) * JsTimeoutMultiplier)
PoolingJsVmConcurrency = opts.JsConcurrency
PoolingJsVmConcurrency -= NonPoolingVMConcurrency
return nil
diff --git a/pkg/js/libs/postgres/memo.postgres.go b/pkg/js/libs/postgres/memo.postgres.go
index 786d87133e..9c61356b07 100755
--- a/pkg/js/libs/postgres/memo.postgres.go
+++ b/pkg/js/libs/postgres/memo.postgres.go
@@ -5,7 +5,7 @@ import (
"errors"
"fmt"
- _ "github.com/lib/pq"
+ _ "github.com/projectdiscovery/nuclei/v3/pkg/js/utils/pgwrap"
utils "github.com/projectdiscovery/nuclei/v3/pkg/js/utils"
diff --git a/pkg/js/libs/postgres/postgres.go b/pkg/js/libs/postgres/postgres.go
index 5aea415010..f20ea85a73 100644
--- a/pkg/js/libs/postgres/postgres.go
+++ b/pkg/js/libs/postgres/postgres.go
@@ -9,10 +9,11 @@ import (
"time"
"github.com/go-pg/pg"
- _ "github.com/lib/pq"
"github.com/praetorian-inc/fingerprintx/pkg/plugins"
postgres "github.com/praetorian-inc/fingerprintx/pkg/plugins/services/postgresql"
utils "github.com/projectdiscovery/nuclei/v3/pkg/js/utils"
+ "github.com/projectdiscovery/nuclei/v3/pkg/js/utils/pgwrap"
+ _ "github.com/projectdiscovery/nuclei/v3/pkg/js/utils/pgwrap"
"github.com/projectdiscovery/nuclei/v3/pkg/protocols/common/protocolstate"
)
@@ -100,7 +101,7 @@ func executeQuery(host string, port int, username string, password string, dbNam
target := net.JoinHostPort(host, fmt.Sprintf("%d", port))
connStr := fmt.Sprintf("postgres://%s:%s@%s/%s?sslmode=disable", username, password, target, dbName)
- db, err := sql.Open("postgres", connStr)
+ db, err := sql.Open(pgwrap.PGWrapDriver, connStr)
if err != nil {
return nil, err
}
@@ -152,6 +153,9 @@ func connect(host string, port int, username string, password string, dbName str
User: username,
Password: password,
Database: dbName,
+ Dialer: func(network, addr string) (net.Conn, error) {
+ return protocolstate.Dialer.Dial(context.Background(), network, addr)
+ },
IdleCheckFrequency: -1,
}).WithContext(ctx).WithTimeout(10 * time.Second)
defer db.Close()
diff --git a/pkg/js/utils/pgwrap/pgwrap.go b/pkg/js/utils/pgwrap/pgwrap.go
new file mode 100644
index 0000000000..d1b82f7ab3
--- /dev/null
+++ b/pkg/js/utils/pgwrap/pgwrap.go
@@ -0,0 +1,53 @@
+package pgwrap
+
+import (
+ "context"
+ "database/sql"
+ "database/sql/driver"
+ "net"
+ "time"
+
+ "github.com/lib/pq"
+ "github.com/projectdiscovery/fastdialer/fastdialer"
+ "github.com/projectdiscovery/nuclei/v3/pkg/protocols/common/protocolstate"
+)
+
+const (
+ PGWrapDriver = "pgwrap"
+)
+
+type pgDial struct {
+ fd *fastdialer.Dialer
+}
+
+func (p *pgDial) Dial(network, address string) (net.Conn, error) {
+ return p.fd.Dial(context.TODO(), network, address)
+}
+
+func (p *pgDial) DialTimeout(network, address string, timeout time.Duration) (net.Conn, error) {
+ ctx, cancel := context.WithTimeoutCause(context.Background(), timeout, fastdialer.ErrDialTimeout)
+ defer cancel()
+ return p.fd.Dial(ctx, network, address)
+}
+
+func (p *pgDial) DialContext(ctx context.Context, network, address string) (net.Conn, error) {
+ return p.fd.Dial(ctx, network, address)
+}
+
+// Unfortunately lib/pq does not provide easy to customize or
+// replace dialer so we need to hijack it by wrapping it in our own
+// driver and register it as postgres driver
+
+// PgDriver is the Postgres database driver.
+type PgDriver struct{}
+
+// Open opens a new connection to the database. name is a connection string.
+// Most users should only use it through database/sql package from the standard
+// library.
+func (d PgDriver) Open(name string) (driver.Conn, error) {
+ return pq.DialOpen(&pgDial{fd: protocolstate.Dialer}, name)
+}
+
+func init() {
+ sql.Register(PGWrapDriver, &PgDriver{})
+}
diff --git a/pkg/output/output.go b/pkg/output/output.go
index 044d164f1f..1ee710ff3b 100644
--- a/pkg/output/output.go
+++ b/pkg/output/output.go
@@ -4,6 +4,7 @@ import (
"encoding/base64"
"fmt"
"io"
+ "log/slog"
"os"
"path/filepath"
"regexp"
@@ -27,9 +28,12 @@ import (
"github.com/projectdiscovery/nuclei/v3/pkg/operators"
protocolUtils "github.com/projectdiscovery/nuclei/v3/pkg/protocols/utils"
"github.com/projectdiscovery/nuclei/v3/pkg/types"
+ "github.com/projectdiscovery/nuclei/v3/pkg/types/nucleierr"
"github.com/projectdiscovery/nuclei/v3/pkg/utils"
+ "github.com/projectdiscovery/utils/errkit"
fileutil "github.com/projectdiscovery/utils/file"
osutils "github.com/projectdiscovery/utils/os"
+ urlutil "github.com/projectdiscovery/utils/url"
)
// Writer is an interface which writes output to somewhere for nuclei events.
@@ -299,10 +303,13 @@ func (w *StandardWriter) Write(event *ResultEvent) error {
// JSONLogRequest is a trace/error log request written to file
type JSONLogRequest struct {
- Template string `json:"template"`
- Input string `json:"input"`
- Error string `json:"error"`
- Type string `json:"type"`
+ Template string `json:"template"`
+ Type string `json:"type"`
+ Input string `json:"input"`
+ Address string `json:"address"`
+ Error string `json:"error"`
+ Kind string `json:"kind,omitempty"`
+ Attrs interface{} `json:"attrs,omitempty"`
}
// Request writes a log the requests trace log
@@ -315,12 +322,43 @@ func (w *StandardWriter) Request(templatePath, input, requestType string, reques
Input: input,
Type: requestType,
}
- if unwrappedErr := utils.UnwrapError(requestErr); unwrappedErr != nil {
- request.Error = unwrappedErr.Error()
- } else {
+ parsed, _ := urlutil.ParseAbsoluteURL(input, false)
+ if parsed != nil {
+ request.Address = parsed.Hostname()
+ port := parsed.Port()
+ if port == "" {
+ switch parsed.Scheme {
+ case urlutil.HTTP:
+ port = "80"
+ case urlutil.HTTPS:
+ port = "443"
+ }
+ }
+ request.Address += ":" + port
+ }
+ errX := errkit.FromError(requestErr)
+ if errX == nil {
request.Error = "none"
+ } else {
+ request.Kind = errkit.ErrKindUnknown.String()
+ var cause error
+ if len(errX.Errors()) > 1 {
+ cause = errX.Errors()[0]
+ }
+ if cause == nil {
+ cause = errX
+ }
+ cause = tryParseCause(cause)
+ request.Error = cause.Error()
+ request.Kind = errkit.GetErrorKind(requestErr, nucleierr.ErrTemplateLogic).String()
+ if len(errX.Attrs()) > 0 {
+ request.Attrs = slog.GroupValue(errX.Attrs()...)
+ }
+ }
+ // check if address slog attr is avaiable in error if set use it
+ if val := errkit.GetAttrValue(requestErr, "address"); val.Any() != nil {
+ request.Address = val.String()
}
-
data, err := jsoniter.Marshal(request)
if err != nil {
return
@@ -452,3 +490,24 @@ func (w *StandardWriter) WriteStoreDebugData(host, templateID, eventType string,
}
}
+
+// tryParseCause tries to parse the cause of given error
+// this is legacy support due to use of errorutil in existing libraries
+// but this should not be required once all libraries are updated
+func tryParseCause(err error) error {
+ if err == nil {
+ return nil
+ }
+ msg := err.Error()
+ if strings.HasPrefix(msg, "ReadStatusLine:") {
+ // last index is actual error (from rawhttp)
+ parts := strings.Split(msg, ":")
+ return errkit.New(strings.TrimSpace(parts[len(parts)-1]))
+ }
+ if strings.Contains(msg, "read ") {
+ // same here
+ parts := strings.Split(msg, ":")
+ return errkit.New(strings.TrimSpace(parts[len(parts)-1]))
+ }
+ return err
+}
diff --git a/pkg/output/output_test.go b/pkg/output/output_test.go
index 7c96c05297..a21f7e8898 100644
--- a/pkg/output/output_test.go
+++ b/pkg/output/output_test.go
@@ -30,7 +30,7 @@ func TestStandardWriterRequest(t *testing.T) {
require.NoError(t, err)
w.Request("path", "input", "http", nil)
- require.Equal(t, `{"template":"path","input":"input","error":"none","type":"http"}`, traceWriter.String())
+ require.Equal(t, `{"template":"path","type":"http","input":"input","address":"input:","error":"none"}`, traceWriter.String())
require.Empty(t, errorWriter.String())
})
@@ -47,7 +47,7 @@ func TestStandardWriterRequest(t *testing.T) {
fmt.Errorf("GET https://example.com/tcpconfig.html/tcpconfig.html giving up after 2 attempts: %w", errors.New("context deadline exceeded (Client.Timeout exceeded while awaiting headers)")),
)
- require.Equal(t, `{"template":"misconfiguration/tcpconfig.yaml","input":"https://example.com/tcpconfig.html","error":"context deadline exceeded (Client.Timeout exceeded while awaiting headers)","type":"http"}`, errorWriter.String())
+ require.Equal(t, `{"template":"misconfiguration/tcpconfig.yaml","type":"http","input":"https://example.com/tcpconfig.html","address":"example.com:443","error":"context deadline exceeded (Client.Timeout exceeded while awaiting headers)","kind":"unknown-error"}`, errorWriter.String())
})
}
diff --git a/pkg/protocols/code/code.go b/pkg/protocols/code/code.go
index 70dda72548..f103f50d0c 100644
--- a/pkg/protocols/code/code.go
+++ b/pkg/protocols/code/code.go
@@ -32,6 +32,7 @@ import (
templateTypes "github.com/projectdiscovery/nuclei/v3/pkg/templates/types"
"github.com/projectdiscovery/nuclei/v3/pkg/types"
contextutil "github.com/projectdiscovery/utils/context"
+ "github.com/projectdiscovery/utils/errkit"
errorutil "github.com/projectdiscovery/utils/errors"
)
@@ -41,7 +42,10 @@ const (
)
var (
+ // pythonEnvRegexCompiled is the compiled regex for python environment variables
pythonEnvRegexCompiled = regexp.MustCompile(pythonEnvRegex)
+ // ErrCodeExecutionDeadline is the error returned when alloted time for script execution exceeds
+ ErrCodeExecutionDeadline = errkit.New("code execution deadline exceeded").SetKind(errkit.ErrKindDeadline).Build()
)
// Request is a request for the SSL protocol
@@ -214,7 +218,7 @@ func (request *Request) ExecuteWithResults(input *contextargs.Context, dynamicVa
}
}
- ctx, cancel := context.WithTimeout(context.Background(), time.Duration(timeout)*time.Second)
+ ctx, cancel := context.WithTimeoutCause(input.Context(), time.Duration(timeout)*time.Second, ErrCodeExecutionDeadline)
defer cancel()
// Note: we use contextutil despite the fact that gozero accepts context as argument
gOutput, err := contextutil.ExecFuncWithTwoReturns(ctx, func() (*gozerotypes.Result, error) {
diff --git a/pkg/protocols/common/contextargs/contextargs.go b/pkg/protocols/common/contextargs/contextargs.go
index 8e8f0361c4..ed004ae370 100644
--- a/pkg/protocols/common/contextargs/contextargs.go
+++ b/pkg/protocols/common/contextargs/contextargs.go
@@ -37,6 +37,13 @@ func New(ctx context.Context) *Context {
return NewWithInput(ctx, "")
}
+// NewWithMetaInput creates a new contextargs instance with meta input
+func NewWithMetaInput(ctx context.Context, input *MetaInput) *Context {
+ n := New(ctx)
+ n.MetaInput = input
+ return n
+}
+
// Create a new contextargs instance with input string
func NewWithInput(ctx context.Context, input string) *Context {
jar, err := cookiejar.New(nil)
@@ -174,3 +181,20 @@ func (ctx *Context) Clone() *Context {
}
return newCtx
}
+
+// GetCopyIfHostOutdated returns a new contextargs if the host is outdated
+func GetCopyIfHostOutdated(ctx *Context, url string) *Context {
+ if ctx.MetaInput.Input == "" {
+ newctx := ctx.Clone()
+ newctx.MetaInput.Input = url
+ return newctx
+ }
+ orig, _ := urlutil.Parse(ctx.MetaInput.Input)
+ newURL, _ := urlutil.Parse(url)
+ if orig != nil && newURL != nil && orig.Host != newURL.Host {
+ newCtx := ctx.Clone()
+ newCtx.MetaInput.Input = newURL.Host
+ return newCtx
+ }
+ return ctx
+}
diff --git a/pkg/protocols/common/contextargs/metainput.go b/pkg/protocols/common/contextargs/metainput.go
index afda2fda26..72eac6247d 100644
--- a/pkg/protocols/common/contextargs/metainput.go
+++ b/pkg/protocols/common/contextargs/metainput.go
@@ -4,11 +4,13 @@ import (
"bytes"
"crypto/md5"
"fmt"
+ "net"
"strings"
jsoniter "github.com/json-iterator/go"
"github.com/projectdiscovery/nuclei/v3/pkg/input/types"
urlutil "github.com/projectdiscovery/utils/url"
+ "github.com/segmentio/ksuid"
)
// MetaInput represents a target with metadata (TODO: replace with https://github.com/projectdiscovery/metainput)
@@ -47,6 +49,56 @@ func (metaInput *MetaInput) URL() (*urlutil.URL, error) {
return instance, nil
}
+// Port returns the port of the target
+// if port is not present then empty string is returned
+func (metaInput *MetaInput) Port() string {
+ target, err := urlutil.ParseAbsoluteURL(metaInput.Input, false)
+ if err != nil {
+ return ""
+ }
+ return target.Port()
+}
+
+// Address return the remote address of target
+// Note: it does not resolve the domain to ip
+// it is meant to be used by hosterrorsCache if invalid metainput
+// is provided then it uses a random ksuid as hostname to avoid skipping valid targets
+func (metaInput *MetaInput) Address() string {
+ var hostname, port string
+ target, err := urlutil.ParseAbsoluteURL(metaInput.Target(), false)
+ if err != nil {
+ if metaInput.CustomIP == "" {
+ // since this is used in hosterrorscache we add a random id
+ // which will never be used to avoid skipping valid targets
+ hostname = fmt.Sprintf("invalid-%s", ksuid.New().String())
+ }
+ } else {
+ hostname = target.Hostname()
+ port = target.Port()
+ if port == "" {
+ switch target.Scheme {
+ case urlutil.HTTP:
+ port = "80"
+ case urlutil.HTTPS:
+ port = "443"
+ default:
+ port = "80"
+ }
+ }
+ }
+ if metaInput.CustomIP != "" {
+ hostname = metaInput.CustomIP
+ }
+ if port == "" {
+ if strings.HasPrefix(hostname, "http://") {
+ port = "80"
+ } else if strings.HasPrefix(hostname, "https://") {
+ port = "443"
+ }
+ }
+ return net.JoinHostPort(hostname, port)
+}
+
// ID returns a unique id/hash for metainput
func (metaInput *MetaInput) ID() string {
if metaInput.CustomIP != "" {
diff --git a/pkg/protocols/common/hosterrorscache/hosterrorscache.go b/pkg/protocols/common/hosterrorscache/hosterrorscache.go
index 3abd1ec55f..e40f13dde3 100644
--- a/pkg/protocols/common/hosterrorscache/hosterrorscache.go
+++ b/pkg/protocols/common/hosterrorscache/hosterrorscache.go
@@ -10,17 +10,25 @@ import (
"github.com/bluele/gcache"
"github.com/projectdiscovery/gologger"
+ "github.com/projectdiscovery/nuclei/v3/pkg/catalog/config"
+ "github.com/projectdiscovery/nuclei/v3/pkg/protocols/common/contextargs"
+ "github.com/projectdiscovery/nuclei/v3/pkg/types/nucleierr"
+ "github.com/projectdiscovery/utils/errkit"
)
// CacheInterface defines the signature of the hosterrorscache so that
// users of Nuclei as embedded lib may implement their own cache
type CacheInterface interface {
- SetVerbose(verbose bool) // log verbosely
- Close() // close the cache
- Check(value string) bool // return true if the host should be skipped
- MarkFailed(value string, err error) // record a failure (and cause) for the host
+ SetVerbose(verbose bool) // log verbosely
+ Close() // close the cache
+ Check(ctx *contextargs.Context) bool // return true if the host should be skipped
+ MarkFailed(ctx *contextargs.Context, err error) // record a failure (and cause) for the host
}
+var (
+ _ CacheInterface = (*Cache)(nil)
+)
+
// Cache is a cache for host based errors. It allows skipping
// certain hosts based on an error threshold.
//
@@ -34,8 +42,10 @@ type Cache struct {
}
type cacheItem struct {
- errors atomic.Int32
sync.Once
+ errors atomic.Int32
+ isPermanentErr bool
+ cause error // optional cause
}
const DefaultMaxHostsCount = 10000
@@ -55,6 +65,16 @@ func (c *Cache) SetVerbose(verbose bool) {
// Close closes the host errors cache
func (c *Cache) Close() {
+ if config.DefaultConfig.IsDebugArgEnabled(config.DebugArgHostErrorStats) {
+ items := c.failedTargets.GetALL(false)
+ for k, v := range items {
+ val, ok := v.(*cacheItem)
+ if !ok {
+ continue
+ }
+ gologger.Info().Label("MaxHostErrorStats").Msgf("Host: %s, Errors: %d", k, val.errors.Load())
+ }
+ }
c.failedTargets.Purge()
}
@@ -88,14 +108,19 @@ func (c *Cache) normalizeCacheValue(value string) string {
// - URL: https?:// type
// - Host:port type
// - host type
-func (c *Cache) Check(value string) bool {
- finalValue := c.normalizeCacheValue(value)
+func (c *Cache) Check(ctx *contextargs.Context) bool {
+ finalValue := c.GetKeyFromContext(ctx, nil)
existingCacheItem, err := c.failedTargets.GetIFPresent(finalValue)
if err != nil {
return false
}
existingCacheItemValue := existingCacheItem.(*cacheItem)
+ if existingCacheItemValue.isPermanentErr {
+ // skipping permanent errors is expected so verbose instead of info
+ gologger.Verbose().Msgf("Skipped %s from target list as found unresponsive permanently: %s", finalValue, existingCacheItemValue.cause)
+ return true
+ }
if existingCacheItemValue.errors.Load() >= int32(c.MaxHostError) {
existingCacheItemValue.Do(func() {
@@ -107,15 +132,22 @@ func (c *Cache) Check(value string) bool {
}
// MarkFailed marks a host as failed previously
-func (c *Cache) MarkFailed(value string, err error) {
+func (c *Cache) MarkFailed(ctx *contextargs.Context, err error) {
if !c.checkError(err) {
return
}
- finalValue := c.normalizeCacheValue(value)
+ finalValue := c.GetKeyFromContext(ctx, err)
existingCacheItem, err := c.failedTargets.GetIFPresent(finalValue)
if err != nil || existingCacheItem == nil {
newItem := &cacheItem{errors: atomic.Int32{}}
newItem.errors.Store(1)
+ if errkit.IsKind(err, errkit.ErrKindNetworkPermanent) {
+ // skip this address altogether
+ // permanent errors are always permanent hence this is created once
+ // and never updated so no need to synchronize
+ newItem.isPermanentErr = true
+ newItem.cause = err
+ }
_ = c.failedTargets.Set(finalValue, newItem)
return
}
@@ -124,19 +156,77 @@ func (c *Cache) MarkFailed(value string, err error) {
_ = c.failedTargets.Set(finalValue, existingCacheItemValue)
}
-var reCheckError = regexp.MustCompile(`(no address found for host|Client\.Timeout exceeded while awaiting headers|could not resolve host|connection refused|connection reset by peer|i/o timeout|could not connect to any address found for host|timeout awaiting response headers)`)
+// GetKeyFromContext returns the key for the cache from the context
+func (c *Cache) GetKeyFromContext(ctx *contextargs.Context, err error) string {
+ // Note:
+ // ideally any changes made to remote addr in template like {{Hostname}}:81 etc
+ // should be reflected in contextargs but it is not yet reflected in some cases
+ // and needs refactor of ScanContext + ContextArgs to achieve that
+ // i.e why we use real address from error if present
+ address := ctx.MetaInput.Address()
+ // get address override from error
+ if err != nil {
+ tmp := errkit.GetAttrValue(err, "address")
+ if tmp.Any() != nil {
+ address = tmp.String()
+ }
+ }
+ finalValue := c.normalizeCacheValue(address)
+ return finalValue
+}
+
+var reCheckError = regexp.MustCompile(`(no address found for host|could not resolve host|connection refused|connection reset by peer|could not connect to any address found for host|timeout awaiting response headers)`)
// checkError checks if an error represents a type that should be
// added to the host skipping table.
+// it first parses error and extracts the cause and checks for blacklisted
+// or common errors that should be skipped
func (c *Cache) checkError(err error) bool {
if err == nil {
return false
}
- errString := err.Error()
- for _, msg := range c.TrackError {
- if strings.Contains(errString, msg) {
- return true
+ kind := errkit.GetErrorKind(err, nucleierr.ErrTemplateLogic)
+ switch kind {
+ case nucleierr.ErrTemplateLogic:
+ // these are errors that are not related to the target
+ // and are due to template logic
+ return false
+ case errkit.ErrKindNetworkTemporary:
+ // these should not be counted as host errors
+ return false
+ case errkit.ErrKindNetworkPermanent:
+ // these should be counted as host errors
+ return true
+ case errkit.ErrKindDeadline:
+ // these should not be counted as host errors
+ return false
+ default:
+ // parse error for furthur processing
+ errX := errkit.FromError(err)
+ tmp := errX.Cause()
+ cause := tmp.Error()
+ if strings.Contains(cause, "ReadStatusLine:") && strings.Contains(cause, "read: connection reset by peer") {
+ // this is a FP and should not be counted as a host error
+ // because server closes connection when it reads corrupted bytes which we send via rawhttp
+ return false
+ }
+ if strings.HasPrefix(cause, "ReadStatusLine:") {
+ // error is present in last part when using rawhttp
+ // this will be fixed once errkit is used everywhere
+ lastIndex := strings.LastIndex(cause, ":")
+ if lastIndex == -1 {
+ lastIndex = 0
+ }
+ if lastIndex >= len(cause)-1 {
+ lastIndex = 0
+ }
+ cause = cause[lastIndex+1:]
+ }
+ for _, msg := range c.TrackError {
+ if strings.Contains(cause, msg) {
+ return true
+ }
}
+ return reCheckError.MatchString(cause)
}
- return reCheckError.MatchString(errString)
}
diff --git a/pkg/protocols/common/hosterrorscache/hosterrorscache_test.go b/pkg/protocols/common/hosterrorscache/hosterrorscache_test.go
index a4730366e0..6fd04b475b 100644
--- a/pkg/protocols/common/hosterrorscache/hosterrorscache_test.go
+++ b/pkg/protocols/common/hosterrorscache/hosterrorscache_test.go
@@ -1,11 +1,13 @@
package hosterrorscache
import (
+ "context"
"fmt"
"sync"
"sync/atomic"
"testing"
+ "github.com/projectdiscovery/nuclei/v3/pkg/protocols/common/contextargs"
"github.com/stretchr/testify/require"
)
@@ -13,8 +15,8 @@ func TestCacheCheck(t *testing.T) {
cache := New(3, DefaultMaxHostsCount, nil)
for i := 0; i < 100; i++ {
- cache.MarkFailed("test", fmt.Errorf("could not resolve host"))
- got := cache.Check("test")
+ cache.MarkFailed(newCtxArgs("test"), fmt.Errorf("could not resolve host"))
+ got := cache.Check(newCtxArgs("test"))
if i < 2 {
// till 3 the host is not flagged to skip
require.False(t, got)
@@ -24,7 +26,7 @@ func TestCacheCheck(t *testing.T) {
}
}
- value := cache.Check("test")
+ value := cache.Check(newCtxArgs("test"))
require.Equal(t, true, value, "could not get checked value")
}
@@ -32,8 +34,8 @@ func TestTrackErrors(t *testing.T) {
cache := New(3, DefaultMaxHostsCount, []string{"custom error"})
for i := 0; i < 100; i++ {
- cache.MarkFailed("custom", fmt.Errorf("got: nested: custom error"))
- got := cache.Check("custom")
+ cache.MarkFailed(newCtxArgs("custom"), fmt.Errorf("got: nested: custom error"))
+ got := cache.Check(newCtxArgs("custom"))
if i < 2 {
// till 3 the host is not flagged to skip
require.False(t, got)
@@ -42,7 +44,7 @@ func TestTrackErrors(t *testing.T) {
require.True(t, got)
}
}
- value := cache.Check("custom")
+ value := cache.Check(newCtxArgs("custom"))
require.Equal(t, true, value, "could not get checked value")
}
@@ -73,16 +75,18 @@ func TestCacheMarkFailed(t *testing.T) {
tests := []struct {
host string
- expected int
+ expected int32
}{
{"http://example.com:80", 1},
{"example.com:80", 2},
- {"example.com", 1},
+ // earlier if port is not provided then port was omitted
+ // but from now it will default to appropriate http scheme based port with 80 as default
+ {"example.com:443", 1},
}
for _, test := range tests {
- normalizedCacheValue := cache.normalizeCacheValue(test.host)
- cache.MarkFailed(test.host, fmt.Errorf("no address found for host"))
+ normalizedCacheValue := cache.GetKeyFromContext(newCtxArgs(test.host), nil)
+ cache.MarkFailed(newCtxArgs(test.host), fmt.Errorf("no address found for host"))
failedTarget, err := cache.failedTargets.Get(normalizedCacheValue)
require.Nil(t, err)
require.NotNil(t, failedTarget)
@@ -102,7 +106,7 @@ func TestCacheMarkFailedConcurrent(t *testing.T) {
}{
{"http://example.com:80", 200},
{"example.com:80", 200},
- {"example.com", 100},
+ {"example.com:443", 100},
}
// the cache is not atomic during items creation, so we pre-create them with counter to zero
@@ -120,14 +124,14 @@ func TestCacheMarkFailedConcurrent(t *testing.T) {
wg.Add(1)
go func() {
defer wg.Done()
- cache.MarkFailed(currentTest.host, fmt.Errorf("could not resolve host"))
+ cache.MarkFailed(newCtxArgs(currentTest.host), fmt.Errorf("could not resolve host"))
}()
}
}
wg.Wait()
for _, test := range tests {
- require.True(t, cache.Check(test.host))
+ require.True(t, cache.Check(newCtxArgs(test.host)))
normalizedCacheValue := cache.normalizeCacheValue(test.host)
failedTarget, err := cache.failedTargets.Get(normalizedCacheValue)
@@ -139,3 +143,8 @@ func TestCacheMarkFailedConcurrent(t *testing.T) {
require.EqualValues(t, test.expected, value.errors.Load())
}
}
+
+func newCtxArgs(value string) *contextargs.Context {
+ ctx := contextargs.NewWithInput(context.TODO(), value)
+ return ctx
+}
diff --git a/pkg/protocols/common/protocolinit/init.go b/pkg/protocols/common/protocolinit/init.go
index 2ab5d7ca9c..59f7dd40b0 100644
--- a/pkg/protocols/common/protocolinit/init.go
+++ b/pkg/protocols/common/protocolinit/init.go
@@ -13,7 +13,6 @@ import (
// Init initializes the client pools for the protocols
func Init(options *types.Options) error {
-
if err := protocolstate.Init(options); err != nil {
return err
}
@@ -40,4 +39,5 @@ func Init(options *types.Options) error {
func Close() {
protocolstate.Dialer.Close()
+ protocolstate.Dialer = nil
}
diff --git a/pkg/protocols/common/protocolstate/state.go b/pkg/protocols/common/protocolstate/state.go
index ae5ac6cb9d..89675e7696 100644
--- a/pkg/protocols/common/protocolstate/state.go
+++ b/pkg/protocols/common/protocolstate/state.go
@@ -210,6 +210,7 @@ func interfaceAddresses(interfaceName string) ([]net.Addr, error) {
func Close() {
if Dialer != nil {
Dialer.Close()
+ Dialer = nil
}
StopActiveMemGuardian()
}
diff --git a/pkg/protocols/headless/engine/page_actions.go b/pkg/protocols/headless/engine/page_actions.go
index 348cab0aec..57cc371093 100644
--- a/pkg/protocols/headless/engine/page_actions.go
+++ b/pkg/protocols/headless/engine/page_actions.go
@@ -24,6 +24,7 @@ import (
protocolutils "github.com/projectdiscovery/nuclei/v3/pkg/protocols/utils"
httputil "github.com/projectdiscovery/nuclei/v3/pkg/protocols/utils/http"
contextutil "github.com/projectdiscovery/utils/context"
+ "github.com/projectdiscovery/utils/errkit"
errorutil "github.com/projectdiscovery/utils/errors"
fileutil "github.com/projectdiscovery/utils/file"
folderutil "github.com/projectdiscovery/utils/folder"
@@ -35,6 +36,8 @@ import (
var (
errinvalidArguments = errorutil.New("invalid arguments provided")
ErrLFAccessDenied = errorutil.New("Use -allow-local-file-access flag to enable local file access")
+ // ErrActionExecDealine is the error returned when alloted time for action execution exceeds
+ ErrActionExecDealine = errkit.New("headless action execution deadline exceeded").SetKind(errkit.ErrKindDeadline).Build()
)
const (
@@ -619,7 +622,7 @@ func (p *Page) WaitEvent(act *Action, out map[string]string) (func() error, erro
// Just wait the event to happen
waitFunc := func() (err error) {
// execute actual wait event
- ctx, cancel := context.WithTimeout(context.Background(), maxDuration)
+ ctx, cancel := context.WithTimeoutCause(context.Background(), maxDuration, ErrActionExecDealine)
defer cancel()
err = contextutil.ExecFunc(ctx, p.page.WaitEvent(waitEvent))
return
diff --git a/pkg/protocols/http/httpclientpool/clientpool.go b/pkg/protocols/http/httpclientpool/clientpool.go
index 3e2baf55a5..4200e25b78 100644
--- a/pkg/protocols/http/httpclientpool/clientpool.go
+++ b/pkg/protocols/http/httpclientpool/clientpool.go
@@ -35,9 +35,10 @@ var (
forceMaxRedirects int
normalClient *retryablehttp.Client
clientPool *mapsutil.SyncLockMap[string, *retryablehttp.Client]
- // ResponseHeaderTimeout is the timeout for response headers
+ // MaxResponseHeaderTimeout is the timeout for response headers
// to be read from the server (this prevents infinite hang started by server if any)
- ResponseHeaderTimeout = time.Duration(5) * time.Second
+ // Note: this will be overridden temporarily when using @timeout request annotation
+ MaxResponseHeaderTimeout = time.Duration(10) * time.Second
// HttpTimeoutMultiplier is the multiplier for the http timeout
HttpTimeoutMultiplier = 3
)
@@ -53,6 +54,9 @@ func Init(options *types.Options) error {
if normalClient != nil {
return nil
}
+ if options.Timeout > 10 {
+ MaxResponseHeaderTimeout = time.Duration(options.Timeout) * time.Second
+ }
if options.ShouldFollowHTTPRedirects() {
forceMaxRedirects = options.MaxRedirects
}
@@ -111,6 +115,8 @@ type Configuration struct {
RedirectFlow RedirectFlow
// Connection defines custom connection configuration
Connection *ConnectionConfiguration
+ // ResponseHeaderTimeout is the timeout for response body to be read from the server
+ ResponseHeaderTimeout time.Duration
}
// Hash returns the hash of the configuration to allow client pooling
@@ -129,13 +135,15 @@ func (c *Configuration) Hash() string {
builder.WriteString(strconv.FormatBool(c.DisableCookie))
builder.WriteString("c")
builder.WriteString(strconv.FormatBool(c.Connection != nil))
+ builder.WriteString("r")
+ builder.WriteString(strconv.FormatInt(int64(c.ResponseHeaderTimeout.Seconds()), 10))
hash := builder.String()
return hash
}
// HasStandardOptions checks whether the configuration requires custom settings
func (c *Configuration) HasStandardOptions() bool {
- return c.Threads == 0 && c.MaxRedirects == 0 && c.RedirectFlow == DontFollowRedirect && c.DisableCookie && c.Connection == nil && !c.NoTimeout
+ return c.Threads == 0 && c.MaxRedirects == 0 && c.RedirectFlow == DontFollowRedirect && c.DisableCookie && c.Connection == nil && !c.NoTimeout && c.ResponseHeaderTimeout == 0
}
// GetRawHTTP returns the rawhttp request client
@@ -182,6 +190,9 @@ func wrappedGet(options *types.Options, configuration *Configuration) (*retryabl
maxIdleConns := 0
maxConnsPerHost := 0
maxIdleConnsPerHost := -1
+ // do not split given timeout into chunks for retry
+ // because this won't work on slow hosts
+ retryableHttpOptions.NoAdjustTimeout = true
if configuration.Threads > 0 || options.ScanStrategy == scanstrategy.HostSpray.String() {
// Single host
@@ -235,6 +246,12 @@ func wrappedGet(options *types.Options, configuration *Configuration) (*retryabl
return nil, errors.Wrap(err, "could not create client certificate")
}
+ // responseHeaderTimeout is max timeout for response headers to be read
+ responseHeaderTimeout := MaxResponseHeaderTimeout
+ if configuration.ResponseHeaderTimeout != 0 {
+ responseHeaderTimeout = configuration.ResponseHeaderTimeout
+ }
+
transport := &http.Transport{
ForceAttemptHTTP2: options.ForceAttemptHTTP2,
DialContext: Dialer.Dial,
@@ -252,7 +269,7 @@ func wrappedGet(options *types.Options, configuration *Configuration) (*retryabl
MaxConnsPerHost: maxConnsPerHost,
TLSClientConfig: tlsConfig,
DisableKeepAlives: disableKeepAlives,
- ResponseHeaderTimeout: ResponseHeaderTimeout,
+ ResponseHeaderTimeout: responseHeaderTimeout,
}
if types.ProxyURL != "" {
@@ -351,7 +368,7 @@ func makeCheckRedirectFunc(redirectType RedirectFlow, maxRedirects int) checkRed
}
}
-func checkMaxRedirects(req *http.Request, via []*http.Request, maxRedirects int) error {
+func checkMaxRedirects(_ *http.Request, via []*http.Request, maxRedirects int) error {
if maxRedirects == 0 {
if len(via) > defaultMaxRedirects {
return http.ErrUseLastResponse
diff --git a/pkg/protocols/http/httpclientpool/options.go b/pkg/protocols/http/httpclientpool/options.go
new file mode 100644
index 0000000000..b6e60b6367
--- /dev/null
+++ b/pkg/protocols/http/httpclientpool/options.go
@@ -0,0 +1,54 @@
+package httpclientpool
+
+import (
+ "io"
+ "net/http"
+ "strings"
+ "time"
+
+ "github.com/projectdiscovery/rawhttp"
+)
+
+// WithCustomTimeout is a configuration for custom timeout
+type WithCustomTimeout struct {
+ Timeout time.Duration
+}
+
+// RawHttpRequestOpts is a configuration for raw http request
+type RawHttpRequestOpts struct {
+ // Method is the http method to use
+ Method string
+ // URL is the url to request
+ URL string
+ // Path is request path to use
+ Path string
+ // Headers is the headers to use
+ Headers map[string][]string
+ // Body is the body to use
+ Body io.Reader
+ // Options is more client related options
+ Options *rawhttp.Options
+}
+
+// SendRawRequest sends a raw http request with the provided options and returns http response
+func SendRawRequest(client *rawhttp.Client, opts *RawHttpRequestOpts) (*http.Response, error) {
+ resp, err := client.DoRawWithOptions(opts.Method, opts.URL, opts.Path, opts.Headers, opts.Body, opts.Options)
+ if err != nil {
+ cause := err.Error()
+ if strings.Contains(cause, "ReadStatusLine: ") && strings.Contains(cause, "read: connection reset by peer") {
+ // this error is caused when rawhttp client sends a corrupted or malformed request packet to server
+ // some servers may attempt gracefully shutdown but most will just abruptly close the connection which results
+ // in a connection reset by peer error and this can be safely assumed as 400 Bad Request in terms of normal http flow
+ req, _ := http.NewRequest(opts.Method, opts.URL, opts.Body)
+ req.Header = opts.Headers
+ resp = &http.Response{
+ Request: req,
+ StatusCode: http.StatusBadRequest,
+ Status: http.StatusText(http.StatusBadRequest),
+ Body: io.NopCloser(strings.NewReader("")),
+ }
+ return resp, nil
+ }
+ }
+ return resp, err
+}
diff --git a/pkg/protocols/http/request.go b/pkg/protocols/http/request.go
index c372e3262b..7b6c6cee24 100644
--- a/pkg/protocols/http/request.go
+++ b/pkg/protocols/http/request.go
@@ -35,8 +35,10 @@ import (
"github.com/projectdiscovery/nuclei/v3/pkg/protocols/http/signerpool"
templateTypes "github.com/projectdiscovery/nuclei/v3/pkg/templates/types"
"github.com/projectdiscovery/nuclei/v3/pkg/types"
+ "github.com/projectdiscovery/nuclei/v3/pkg/types/nucleierr"
"github.com/projectdiscovery/rawhttp"
convUtil "github.com/projectdiscovery/utils/conversion"
+ "github.com/projectdiscovery/utils/errkit"
errorutil "github.com/projectdiscovery/utils/errors"
httpUtils "github.com/projectdiscovery/utils/http"
"github.com/projectdiscovery/utils/reader"
@@ -55,7 +57,9 @@ const (
var (
MaxBodyRead = int64(10 * 1024 * 1024) // 10MB
// ErrMissingVars is error occured when variables are missing
- ErrMissingVars = errors.New("stop execution due to unresolved variables")
+ ErrMissingVars = errkit.New("stop execution due to unresolved variables").SetKind(nucleierr.ErrTemplateLogic).Build()
+ // ErrHttpEngineRequestDeadline is error occured when request deadline set by http request engine is exceeded
+ ErrHttpEngineRequestDeadline = errkit.New("http request engine deadline exceeded").SetKind(errkit.ErrKindDeadline).Build()
)
// Type returns the type of the protocol request
@@ -147,15 +151,15 @@ func (request *Request) executeRaceRequest(input *contextargs.Context, previous
return
}
// marks thsi host as unresponsive if applicable
- request.markUnresponsiveHost(input, err)
- if request.isUnresponsiveHost(input) {
+ request.markUnresponsiveAddress(input, err)
+ if request.isUnresponsiveAddress(input) {
// stop all inflight requests
spmHandler.Cancel()
}
})
for i := 0; i < request.RaceNumberRequests; i++ {
- if spmHandler.FoundFirstMatch() || request.isUnresponsiveHost(input) {
+ if spmHandler.FoundFirstMatch() || request.isUnresponsiveAddress(input) {
// stop sending more requests condition is met
break
}
@@ -163,7 +167,7 @@ func (request *Request) executeRaceRequest(input *contextargs.Context, previous
// execute http request
go func(httpRequest *generatedRequest) {
defer spmHandler.Release()
- if spmHandler.FoundFirstMatch() || request.isUnresponsiveHost(input) {
+ if spmHandler.FoundFirstMatch() || request.isUnresponsiveAddress(input) {
// stop sending more requests condition is met
return
}
@@ -232,8 +236,8 @@ func (request *Request) executeParallelHTTP(input *contextargs.Context, dynamicV
return
}
// marks thsi host as unresponsive if applicable
- request.markUnresponsiveHost(input, err)
- if request.isUnresponsiveHost(input) {
+ request.markUnresponsiveAddress(input, err)
+ if request.isUnresponsiveAddress(input) {
// stop all inflight requests
spmHandler.Cancel()
}
@@ -261,7 +265,7 @@ func (request *Request) executeParallelHTTP(input *contextargs.Context, dynamicV
}
// break if stop at first match is found or host is unresponsive
- if spmHandler.FoundFirstMatch() || request.isUnresponsiveHost(input) {
+ if spmHandler.FoundFirstMatch() || request.isUnresponsiveAddress(input) {
break
}
@@ -277,17 +281,23 @@ func (request *Request) executeParallelHTTP(input *contextargs.Context, dynamicV
if input.MetaInput.Input == "" {
input.MetaInput.Input = generatedHttpRequest.URL()
}
+ updatedInput := contextargs.GetCopyIfHostOutdated(input, generatedHttpRequest.URL())
+ if request.isUnresponsiveAddress(updatedInput) {
+ // skip on unresponsive host no need to continue
+ spmHandler.Cancel()
+ return nil
+ }
spmHandler.Acquire()
go func(httpRequest *generatedRequest) {
defer spmHandler.Release()
- if spmHandler.FoundFirstMatch() || request.isUnresponsiveHost(input) || spmHandler.Cancelled() {
+ if spmHandler.FoundFirstMatch() || request.isUnresponsiveAddress(updatedInput) || spmHandler.Cancelled() {
return
}
// putting ratelimiter here prevents any unnecessary waiting if any
request.options.RateLimitTake()
// after ratelimit take, check if we need to stop
- if spmHandler.FoundFirstMatch() || request.isUnresponsiveHost(input) || spmHandler.Cancelled() {
+ if spmHandler.FoundFirstMatch() || request.isUnresponsiveAddress(updatedInput) || spmHandler.Cancelled() {
return
}
@@ -370,8 +380,8 @@ func (request *Request) executeTurboHTTP(input *contextargs.Context, dynamicValu
return
}
// marks thsi host as unresponsive if applicable
- request.markUnresponsiveHost(input, err)
- if request.isUnresponsiveHost(input) {
+ request.markUnresponsiveAddress(input, err)
+ if request.isUnresponsiveAddress(input) {
// stop all inflight requests
spmHandler.Cancel()
}
@@ -389,7 +399,7 @@ func (request *Request) executeTurboHTTP(input *contextargs.Context, dynamicValu
default:
}
- if spmHandler.FoundFirstMatch() || request.isUnresponsiveHost(input) || spmHandler.Cancelled() {
+ if spmHandler.FoundFirstMatch() || request.isUnresponsiveAddress(input) || spmHandler.Cancelled() {
// skip if first match is found
break
}
@@ -403,11 +413,17 @@ func (request *Request) executeTurboHTTP(input *contextargs.Context, dynamicValu
if input.MetaInput.Input == "" {
input.MetaInput.Input = generatedHttpRequest.URL()
}
+ updatedInput := contextargs.GetCopyIfHostOutdated(input, generatedHttpRequest.URL())
+ if request.isUnresponsiveAddress(updatedInput) {
+ // skip on unresponsive host no need to continue
+ spmHandler.Cancel()
+ return nil
+ }
generatedHttpRequest.pipelinedClient = pipeClient
spmHandler.Acquire()
go func(httpRequest *generatedRequest) {
defer spmHandler.Release()
- if spmHandler.FoundFirstMatch() || request.isUnresponsiveHost(input) {
+ if spmHandler.FoundFirstMatch() || request.isUnresponsiveAddress(updatedInput) {
// skip if first match is found
return
}
@@ -466,7 +482,7 @@ func (request *Request) ExecuteWithResults(input *contextargs.Context, dynamicVa
request.options.RateLimitTake()
ctx := request.newContext(input)
- ctxWithTimeout, cancel := context.WithTimeout(ctx, httpclientpool.GetHttpTimeout(request.options.Options))
+ ctxWithTimeout, cancel := context.WithTimeoutCause(ctx, httpclientpool.GetHttpTimeout(request.options.Options), ErrHttpEngineRequestDeadline)
defer cancel()
generatedHttpRequest, err := generator.Make(ctxWithTimeout, input, data, payloads, dynamicValue)
@@ -477,6 +493,12 @@ func (request *Request) ExecuteWithResults(input *contextargs.Context, dynamicVa
request.options.Progress.IncrementFailedRequestsBy(int64(generator.Total()))
return true, err
}
+ // ideally if http template used a custom port or hostname
+ // we would want to update it in input but currently templateCtx logic
+ // is closely tied to contextargs.Context so we are temporarily creating
+ // a copy and using it to check for host errors etc
+ // but this should be replaced once templateCtx is refactored properly
+ updatedInput := contextargs.GetCopyIfHostOutdated(input, generatedHttpRequest.URL())
if generatedHttpRequest.customCancelFunction != nil {
defer generatedHttpRequest.customCancelFunction()
@@ -487,7 +509,7 @@ func (request *Request) ExecuteWithResults(input *contextargs.Context, dynamicVa
input.MetaInput.Input = generatedHttpRequest.URL()
}
// Check if hosts keep erroring
- if request.isUnresponsiveHost(input) {
+ if request.isUnresponsiveAddress(updatedInput) {
return true, nil
}
var gotMatches bool
@@ -527,7 +549,7 @@ func (request *Request) ExecuteWithResults(input *contextargs.Context, dynamicVa
}
if execReqErr != nil {
// if applicable mark the host as unresponsive
- request.markUnresponsiveHost(input, execReqErr)
+ request.markUnresponsiveAddress(updatedInput, execReqErr)
requestErr = errorutil.NewWithErr(execReqErr).Msgf("got err while executing %v", generatedHttpRequest.URL())
request.options.Progress.IncrementFailedRequestsBy(1)
} else {
@@ -580,7 +602,7 @@ const drainReqSize = int64(8 * 1024)
// executeRequest executes the actual generated request and returns error if occurred
func (request *Request) executeRequest(input *contextargs.Context, generatedRequest *generatedRequest, previousEvent output.InternalEvent, hasInteractMatchers bool, processEvent protocols.OutputEventCallback, requestCount int) (err error) {
// Check if hosts keep erroring
- if request.isUnresponsiveHost(input) {
+ if request.isUnresponsiveAddress(input) {
return fmt.Errorf("hostErrorsCache : host %s is unresponsive", input.MetaInput.Input)
}
@@ -713,7 +735,16 @@ func (request *Request) executeRequest(input *contextargs.Context, generatedRequ
inputUrl = url.String()
}
formedURL = fmt.Sprintf("%s%s", inputUrl, generatedRequest.rawRequest.Path)
- resp, err = generatedRequest.original.rawhttpClient.DoRawWithOptions(generatedRequest.rawRequest.Method, inputUrl, generatedRequest.rawRequest.Path, generators.ExpandMapValues(generatedRequest.rawRequest.Headers), io.NopCloser(strings.NewReader(generatedRequest.rawRequest.Data)), &options)
+
+ // send rawhttp request and get response
+ resp, err = httpclientpool.SendRawRequest(generatedRequest.original.rawhttpClient, &httpclientpool.RawHttpRequestOpts{
+ Method: generatedRequest.rawRequest.Method,
+ URL: inputUrl,
+ Path: generatedRequest.rawRequest.Path,
+ Headers: generators.ExpandMapValues(generatedRequest.rawRequest.Headers),
+ Body: io.NopCloser(strings.NewReader(generatedRequest.rawRequest.Data)),
+ Options: &options,
+ })
} else {
//** For Normal requests **//
hostname = generatedRequest.request.URL.Host
@@ -731,17 +762,34 @@ func (request *Request) executeRequest(input *contextargs.Context, generatedRequ
if errSignature := request.handleSignature(generatedRequest); errSignature != nil {
return errSignature
}
-
httpclient := request.httpClient
+
+ // this will be assigned/updated if this specific request has a custom configuration
+ var modifiedConfig *httpclientpool.Configuration
+
+ // check for cookie related configuration
if input.CookieJar != nil {
connConfiguration := request.connConfiguration
connConfiguration.Connection.SetCookieJar(input.CookieJar)
- client, err := httpclientpool.Get(request.options.Options, connConfiguration)
+ modifiedConfig = connConfiguration
+ }
+ // check for request updatedTimeout annotation
+ updatedTimeout, ok := generatedRequest.request.Context().Value(httpclientpool.WithCustomTimeout{}).(httpclientpool.WithCustomTimeout)
+ if ok {
+ if modifiedConfig == nil {
+ modifiedConfig = request.connConfiguration
+ }
+ modifiedConfig.ResponseHeaderTimeout = updatedTimeout.Timeout
+ }
+
+ if modifiedConfig != nil {
+ client, err := httpclientpool.Get(request.options.Options, modifiedConfig)
if err != nil {
return errors.Wrap(err, "could not get http client")
}
httpclient = client
}
+
resp, err = httpclient.Do(generatedRequest.request)
}
}
@@ -1087,20 +1135,20 @@ func (request *Request) newContext(input *contextargs.Context) context.Context {
return input.Context()
}
-// markUnresponsiveHost checks if the error is a unreponsive host error and marks it
-func (request *Request) markUnresponsiveHost(input *contextargs.Context, err error) {
+// markUnresponsiveAddress checks if the error is a unreponsive host error and marks it
+func (request *Request) markUnresponsiveAddress(input *contextargs.Context, err error) {
if err == nil {
return
}
if request.options.HostErrorsCache != nil {
- request.options.HostErrorsCache.MarkFailed(input.MetaInput.ID(), err)
+ request.options.HostErrorsCache.MarkFailed(input, err)
}
}
-// isUnresponsiveHost checks if the error is a unreponsive based on its execution history
-func (request *Request) isUnresponsiveHost(input *contextargs.Context) bool {
+// isUnresponsiveAddress checks if the error is a unreponsive based on its execution history
+func (request *Request) isUnresponsiveAddress(input *contextargs.Context) bool {
if request.options.HostErrorsCache != nil {
- return request.options.HostErrorsCache.Check(input.MetaInput.ID())
+ return request.options.HostErrorsCache.Check(input)
}
return false
}
diff --git a/pkg/protocols/http/request_annotations.go b/pkg/protocols/http/request_annotations.go
index 67bf7c1671..c0f4b69774 100644
--- a/pkg/protocols/http/request_annotations.go
+++ b/pkg/protocols/http/request_annotations.go
@@ -10,7 +10,9 @@ import (
"github.com/projectdiscovery/fastdialer/fastdialer"
"github.com/projectdiscovery/nuclei/v3/pkg/protocols/http/httpclientpool"
+ "github.com/projectdiscovery/nuclei/v3/pkg/types/nucleierr"
"github.com/projectdiscovery/retryablehttp-go"
+ "github.com/projectdiscovery/utils/errkit"
iputil "github.com/projectdiscovery/utils/ip"
stringsutil "github.com/projectdiscovery/utils/strings"
)
@@ -27,6 +29,15 @@ var (
reTimeoutAnnotation = regexp.MustCompile(`(?m)^@timeout:\s*(.+)\s*$`)
// @once sets the request to be executed only once for a specific URL
reOnceAnnotation = regexp.MustCompile(`(?m)^@once\s*$`)
+
+ // ErrTimeoutAnnotationDeadline is the error returned when a specific amount of time was exceeded for a request
+ // which was alloted using @timeout annotation this usually means that vulnerability was not found
+ // in rare case it could also happen due to network congestion
+ // the assigned class is TemplateLogic since this in almost every case means that server is not vulnerable
+ ErrTimeoutAnnotationDeadline = errkit.New("timeout annotation deadline exceeded").SetKind(nucleierr.ErrTemplateLogic).Build()
+ // ErrRequestTimeoutDeadline is the error returned when a specific amount of time was exceeded for a request
+ // this happens when the request execution exceeds alloted time
+ ErrRequestTimeoutDeadline = errkit.New("request timeout deadline exceeded when notimeout is set").SetKind(errkit.ErrKindDeadline).Build()
)
type flowMark int
@@ -120,12 +131,16 @@ func (r *Request) parseAnnotations(rawRequest string, request *retryablehttp.Req
value := strings.TrimSpace(duration[1])
if parsed, err := time.ParseDuration(value); err == nil {
//nolint:govet // cancelled automatically by withTimeout
- ctx, overrides.cancelFunc = context.WithTimeout(context.Background(), parsed)
+ // global timeout is overridden by annotation by replacing context
+ ctx, overrides.cancelFunc = context.WithTimeoutCause(context.TODO(), parsed, ErrTimeoutAnnotationDeadline)
+ // add timeout value to context
+ ctx = context.WithValue(ctx, httpclientpool.WithCustomTimeout{}, httpclientpool.WithCustomTimeout{Timeout: parsed})
request = request.Clone(ctx)
}
} else {
//nolint:govet // cancelled automatically by withTimeout
- ctx, overrides.cancelFunc = context.WithTimeout(context.Background(), httpclientpool.GetHttpTimeout(r.options.Options))
+ // global timeout is overridden by annotation by replacing context
+ ctx, overrides.cancelFunc = context.WithTimeoutCause(context.TODO(), httpclientpool.GetHttpTimeout(r.options.Options), ErrRequestTimeoutDeadline)
request = request.Clone(ctx)
}
}
diff --git a/pkg/protocols/http/request_fuzz.go b/pkg/protocols/http/request_fuzz.go
index bc5f9caa9b..8d980b4bf3 100644
--- a/pkg/protocols/http/request_fuzz.go
+++ b/pkg/protocols/http/request_fuzz.go
@@ -6,7 +6,6 @@ package http
// -> request.executeGeneratedFuzzingRequest [execute final generated fuzzing request and get result]
import (
- "context"
"fmt"
"net/http"
"strings"
@@ -133,7 +132,7 @@ func (request *Request) executeAllFuzzingRules(input *contextargs.Context, value
return request.executeGeneratedFuzzingRequest(gr, input, callback)
},
Values: values,
- BaseRequest: baseRequest.Clone(context.TODO()),
+ BaseRequest: baseRequest.Clone(input.Context()),
})
if err == nil {
applicable = true
@@ -158,7 +157,7 @@ func (request *Request) executeAllFuzzingRules(input *contextargs.Context, value
func (request *Request) executeGeneratedFuzzingRequest(gr fuzz.GeneratedRequest, input *contextargs.Context, callback protocols.OutputEventCallback) bool {
hasInteractMatchers := interactsh.HasMatchers(request.CompiledOperators)
hasInteractMarkers := len(gr.InteractURLs) > 0
- if request.options.HostErrorsCache != nil && request.options.HostErrorsCache.Check(input.MetaInput.Input) {
+ if request.options.HostErrorsCache != nil && request.options.HostErrorsCache.Check(input) {
return false
}
request.options.RateLimitTake()
@@ -201,7 +200,7 @@ func (request *Request) executeGeneratedFuzzingRequest(gr fuzz.GeneratedRequest,
}
if requestErr != nil {
if request.options.HostErrorsCache != nil {
- request.options.HostErrorsCache.MarkFailed(input.MetaInput.Input, requestErr)
+ request.options.HostErrorsCache.MarkFailed(input, requestErr)
}
gologger.Verbose().Msgf("[%s] Error occurred in request: %s\n", request.options.TemplateID, requestErr)
}
diff --git a/pkg/protocols/javascript/js.go b/pkg/protocols/javascript/js.go
index 84457b5fca..71f9f53413 100644
--- a/pkg/protocols/javascript/js.go
+++ b/pkg/protocols/javascript/js.go
@@ -32,6 +32,7 @@ import (
protocolutils "github.com/projectdiscovery/nuclei/v3/pkg/protocols/utils"
templateTypes "github.com/projectdiscovery/nuclei/v3/pkg/templates/types"
"github.com/projectdiscovery/nuclei/v3/pkg/types"
+ "github.com/projectdiscovery/utils/errkit"
errorutil "github.com/projectdiscovery/utils/errors"
iputil "github.com/projectdiscovery/utils/ip"
syncutil "github.com/projectdiscovery/utils/sync"
@@ -360,7 +361,7 @@ func (request *Request) ExecuteWithResults(target *contextargs.Context, dynamicV
}
if request.generator != nil && request.Threads > 1 {
- request.executeRequestParallel(context.Background(), hostPort, hostname, input, payloadValues, callback)
+ request.executeRequestParallel(target.Context(), hostPort, hostname, input, payloadValues, callback)
return nil
}
@@ -387,11 +388,10 @@ func (request *Request) ExecuteWithResults(target *contextargs.Context, dynamicV
}
callback(result)
}, requestOptions); err != nil {
- _ = err
- // Review: should we log error here?
- // it is technically not error as it is expected to fail
- // gologger.Warning().Msgf("Could not execute request: %s\n", err)
- // do not return even if error occured
+ if errkit.IsNetworkPermanentErr(err) {
+ // gologger.Verbose().Msgf("Could not execute request: %s\n", err)
+ return err
+ }
}
// If this was a match, and we want to stop at first match, skip all further requests.
shouldStopAtFirstMatch := request.options.Options.StopAtFirstMatch || request.StopAtFirstMatch
@@ -408,8 +408,8 @@ func (request *Request) executeRequestParallel(ctxParent context.Context, hostPo
if threads == 0 {
threads = 1
}
- ctx, cancel := context.WithCancel(ctxParent)
- defer cancel()
+ ctx, cancel := context.WithCancelCause(ctxParent)
+ defer cancel(nil)
requestOptions := request.options
gotmatches := &atomic.Bool{}
@@ -453,16 +453,15 @@ func (request *Request) executeRequestParallel(ctxParent context.Context, hostPo
}
callback(result)
}, requestOptions); err != nil {
- _ = err
- // Review: should we log error here?
- // it is technically not error as it is expected to fail
- // gologger.Warning().Msgf("Could not execute request: %s\n", err)
- // do not return even if error occured
+ if errkit.IsNetworkPermanentErr(err) {
+ cancel(err)
+ return
+ }
}
// If this was a match, and we want to stop at first match, skip all further requests.
if shouldStopAtFirstMatch && gotmatches.Load() {
- cancel()
+ cancel(nil)
return
}
}()
@@ -507,7 +506,6 @@ func (request *Request) executeRequestWithPayloads(hostPort string, input *conte
results = compiler.ExecuteResult{"success": false, "error": err.Error()}
}
request.options.Progress.IncrementRequests()
-
requestOptions.Output.Request(requestOptions.TemplateID, hostPort, request.Type().String(), err)
gologger.Verbose().Msgf("[%s] Sent Javascript request to %s", request.options.TemplateID, hostPort)
diff --git a/pkg/protocols/network/request.go b/pkg/protocols/network/request.go
index 1123f40569..62807abf93 100644
--- a/pkg/protocols/network/request.go
+++ b/pkg/protocols/network/request.go
@@ -1,7 +1,6 @@
package network
import (
- "context"
"encoding/hex"
"fmt"
"net"
@@ -64,7 +63,7 @@ func (request *Request) getOpenPorts(target *contextargs.Context) ([]string, err
errs = append(errs, err)
continue
}
- conn, err := protocolstate.Dialer.Dial(context.TODO(), "tcp", addr)
+ conn, err := protocolstate.Dialer.Dial(target.Context(), "tcp", addr)
if err != nil {
errs = append(errs, err)
continue
@@ -119,6 +118,10 @@ func (request *Request) ExecuteWithResults(target *contextargs.Context, metadata
func (request *Request) executeOnTarget(input *contextargs.Context, visited mapsutil.Map[string, struct{}], metadata, previous output.InternalEvent, callback protocols.OutputEventCallback) error {
var address string
var err error
+ if request.isUnresponsiveAddress(input) {
+ // skip on unresponsive address no need to continue
+ return nil
+ }
if request.SelfContained {
address = ""
@@ -173,6 +176,8 @@ func (request *Request) executeAddress(variables map[string]interface{}, actualA
request.options.Progress.IncrementFailedRequestsBy(1)
return err
}
+ updatedTarget := input.Clone()
+ updatedTarget.MetaInput.Input = actualAddress
// if request threads matches global payload concurrency we follow it
shouldFollowGlobal := request.Threads == request.options.Options.PayloadConcurrency
@@ -206,11 +211,19 @@ func (request *Request) executeAddress(variables map[string]interface{}, actualA
m.Unlock()
}
}
+ if request.isUnresponsiveAddress(updatedTarget) {
+ // skip on unresponsive address no need to continue
+ return nil
+ }
value = generators.MergeMaps(value, payloads)
swg.Add()
go func(vars map[string]interface{}) {
defer swg.Done()
+ if request.isUnresponsiveAddress(updatedTarget) {
+ // skip on unresponsive address no need to continue
+ return
+ }
if err := request.executeRequestWithPayloads(variables, actualAddress, address, input, shouldUseTLS, vars, previous, callback); err != nil {
m.Lock()
multiErr = multierr.Append(multiErr, err)
@@ -240,13 +253,22 @@ func (request *Request) executeRequestWithPayloads(variables map[string]interfac
if host, _, err := net.SplitHostPort(actualAddress); err == nil {
hostname = host
}
+ updatedTarget := input.Clone()
+ updatedTarget.MetaInput.Input = actualAddress
+
+ if request.isUnresponsiveAddress(updatedTarget) {
+ // skip on unresponsive address no need to continue
+ return nil
+ }
if shouldUseTLS {
- conn, err = request.dialer.DialTLS(context.Background(), "tcp", actualAddress)
+ conn, err = request.dialer.DialTLS(input.Context(), "tcp", actualAddress)
} else {
- conn, err = request.dialer.Dial(context.Background(), "tcp", actualAddress)
+ conn, err = request.dialer.Dial(input.Context(), "tcp", actualAddress)
}
if err != nil {
+ // adds it to unresponsive address list if applicable
+ request.markUnresponsiveAddress(updatedTarget, err)
request.options.Output.Request(request.options.TemplatePath, address, request.Type().String(), err)
request.options.Progress.IncrementFailedRequestsBy(1)
return errors.Wrap(err, "could not connect to server")
@@ -475,3 +497,21 @@ func ConnReadNWithTimeout(conn net.Conn, n int64, timeout time.Duration) ([]byte
}
return b[:count], nil
}
+
+// markUnresponsiveAddress checks if the error is a unreponsive host error and marks it
+func (request *Request) markUnresponsiveAddress(input *contextargs.Context, err error) {
+ if err == nil {
+ return
+ }
+ if request.options.HostErrorsCache != nil {
+ request.options.HostErrorsCache.MarkFailed(input, err)
+ }
+}
+
+// isUnresponsiveAddress checks if the error is a unreponsive based on its execution history
+func (request *Request) isUnresponsiveAddress(input *contextargs.Context) bool {
+ if request.options.HostErrorsCache != nil {
+ return request.options.HostErrorsCache.Check(input)
+ }
+ return false
+}
diff --git a/pkg/protocols/websocket/websocket.go b/pkg/protocols/websocket/websocket.go
index 0146b57c49..a956de0e3f 100644
--- a/pkg/protocols/websocket/websocket.go
+++ b/pkg/protocols/websocket/websocket.go
@@ -1,7 +1,6 @@
package websocket
import (
- "context"
"crypto/tls"
"fmt"
"io"
@@ -228,7 +227,7 @@ func (request *Request) executeRequestWithPayloads(target *contextargs.Context,
parsedAddress.Path = path.Join(parsedAddress.Path, parsed.Path)
addressToDial = parsedAddress.String()
- conn, readBuffer, _, err := websocketDialer.Dial(context.Background(), addressToDial)
+ conn, readBuffer, _, err := websocketDialer.Dial(target.Context(), addressToDial)
if err != nil {
requestOptions.Output.Request(requestOptions.TemplateID, input, request.Type().String(), err)
requestOptions.Progress.IncrementFailedRequestsBy(1)
diff --git a/pkg/templates/cluster.go b/pkg/templates/cluster.go
index 97e7b67bd0..603378e986 100644
--- a/pkg/templates/cluster.go
+++ b/pkg/templates/cluster.go
@@ -296,7 +296,7 @@ func (e *ClusterExecuter) Execute(ctx *scan.ScanContext) (bool, error) {
}
})
if err != nil && e.options.HostErrorsCache != nil {
- e.options.HostErrorsCache.MarkFailed(ctx.Input.MetaInput.Input, err)
+ e.options.HostErrorsCache.MarkFailed(ctx.Input, err)
}
return results, err
}
@@ -330,7 +330,7 @@ func (e *ClusterExecuter) ExecuteWithResults(ctx *scan.ScanContext) ([]*output.R
}
if err != nil && e.options.HostErrorsCache != nil {
- e.options.HostErrorsCache.MarkFailed(ctx.Input.MetaInput.Input, err)
+ e.options.HostErrorsCache.MarkFailed(ctx.Input, err)
}
return scanCtx.GenerateResult(), err
}
diff --git a/pkg/testutils/testutils.go b/pkg/testutils/testutils.go
index 68410f7f1d..4ce40ace96 100644
--- a/pkg/testutils/testutils.go
+++ b/pkg/testutils/testutils.go
@@ -20,6 +20,7 @@ import (
"github.com/projectdiscovery/nuclei/v3/pkg/progress"
"github.com/projectdiscovery/nuclei/v3/pkg/protocols"
"github.com/projectdiscovery/nuclei/v3/pkg/protocols/common/protocolinit"
+ "github.com/projectdiscovery/nuclei/v3/pkg/protocols/common/protocolstate"
protocolUtils "github.com/projectdiscovery/nuclei/v3/pkg/protocols/utils"
"github.com/projectdiscovery/nuclei/v3/pkg/types"
"github.com/projectdiscovery/nuclei/v3/pkg/utils"
@@ -27,6 +28,7 @@ import (
// Init initializes the protocols and their configurations
func Init(options *types.Options) {
+ _ = protocolstate.Init(options)
_ = protocolinit.Init(options)
}
diff --git a/pkg/tmplexec/generic/exec.go b/pkg/tmplexec/generic/exec.go
index 29ace4477d..3b337f6b20 100644
--- a/pkg/tmplexec/generic/exec.go
+++ b/pkg/tmplexec/generic/exec.go
@@ -85,7 +85,7 @@ func (g *Generic) ExecuteWithResults(ctx *scan.ScanContext) error {
if err != nil {
ctx.LogError(err)
if g.options.HostErrorsCache != nil {
- g.options.HostErrorsCache.MarkFailed(ctx.Input.MetaInput.ID(), err)
+ g.options.HostErrorsCache.MarkFailed(ctx.Input, err)
}
gologger.Warning().Msgf("[%s] Could not execute request for %s: %s\n", g.options.TemplateID, ctx.Input.MetaInput.PrettyPrint(), err)
}
diff --git a/pkg/types/nucleierr/kinds.go b/pkg/types/nucleierr/kinds.go
new file mode 100644
index 0000000000..9042beaade
--- /dev/null
+++ b/pkg/types/nucleierr/kinds.go
@@ -0,0 +1,28 @@
+package nucleierr
+
+import (
+ "strings"
+
+ "github.com/projectdiscovery/utils/errkit"
+)
+
+var (
+ // ErrTemplateLogic are errors that occured due to missing variable or something similar in template logic
+ // so this is more of a virtual error that is expected due to template logic
+ ErrTemplateLogic = errkit.NewPrimitiveErrKind("TemplateLogic", "Error expected due to template logic", isTemplateLogicKind)
+)
+
+// isTemplateLogicKind checks if an error is of template logic kind
+func isTemplateLogicKind(err *errkit.ErrorX) bool {
+ if err == nil || err.Cause() == nil {
+ return false
+ }
+ v := err.Cause().Error()
+ switch {
+ case strings.Contains(v, "timeout annotation deadline exceeded"):
+ return true
+ case strings.Contains(v, "stop execution due to unresolved variables"):
+ return true
+ }
+ return false
+}
From 8e0bb8fe3407dde569d7b7241dcdec1430a763ff Mon Sep 17 00:00:00 2001
From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com>
Date: Fri, 24 May 2024 19:00:12 +0000
Subject: [PATCH 022/283] chore(deps): bump github.com/projectdiscovery/dsl
from 0.0.56 to 0.0.57
Bumps [github.com/projectdiscovery/dsl](https://github.com/projectdiscovery/dsl) from 0.0.56 to 0.0.57.
- [Release notes](https://github.com/projectdiscovery/dsl/releases)
- [Commits](https://github.com/projectdiscovery/dsl/compare/v0.0.56...v0.0.57)
---
updated-dependencies:
- dependency-name: github.com/projectdiscovery/dsl
dependency-type: direct:production
update-type: version-update:semver-patch
...
Signed-off-by: dependabot[bot]
---
go.mod | 2 +-
go.sum | 4 ++--
2 files changed, 3 insertions(+), 3 deletions(-)
diff --git a/go.mod b/go.mod
index 537cc9ce65..8368b1afd3 100644
--- a/go.mod
+++ b/go.mod
@@ -78,7 +78,7 @@ require (
github.com/mholt/archiver v3.1.1+incompatible
github.com/ory/dockertest/v3 v3.10.0
github.com/praetorian-inc/fingerprintx v1.1.9
- github.com/projectdiscovery/dsl v0.0.56
+ github.com/projectdiscovery/dsl v0.0.57
github.com/projectdiscovery/fasttemplate v0.0.2
github.com/projectdiscovery/go-smb2 v0.0.0-20240129202741-052cc450c6cb
github.com/projectdiscovery/goflags v0.1.52
diff --git a/go.sum b/go.sum
index c176b49953..1c56ada6a6 100644
--- a/go.sum
+++ b/go.sum
@@ -829,8 +829,8 @@ github.com/projectdiscovery/cdncheck v1.0.9 h1:BS15gzj9gb5AVSKqTDzPamfSgStu7nJQO
github.com/projectdiscovery/cdncheck v1.0.9/go.mod h1:18SSl1w7rMj53CGeRIZTbDoa286a6xZIxGbaiEo4Fxs=
github.com/projectdiscovery/clistats v0.0.20 h1:5jO5SLiRJ7f0nDV0ndBNmBeesbROouPooH+DGMgoWq4=
github.com/projectdiscovery/clistats v0.0.20/go.mod h1:GJ2av0KnOvK0AISQnP8hyDclYIji1LVkx2l0pwnzAu4=
-github.com/projectdiscovery/dsl v0.0.56 h1:iVFIfDdGXkrXTh5sf6hRaxqTTEYiv/LnNjoOHJIfHiY=
-github.com/projectdiscovery/dsl v0.0.56/go.mod h1:3UBjPvtiy8c5E8oAUrKobMKjb3kEwQuGB5tqIN964Og=
+github.com/projectdiscovery/dsl v0.0.57 h1:BFjkhtJHzvxSNO6bJJ0UkA7tIwysgsyqWbuhn3FVQfI=
+github.com/projectdiscovery/dsl v0.0.57/go.mod h1:w9XL7AT/RDyg7yiFQN8eWWefXiYQt3mojPDdjyTdQ0k=
github.com/projectdiscovery/fastdialer v0.1.0 h1:NiId7eXvYZjiBo83AnIZ6K8ZjzLJU8wGXdRLzuKcRrg=
github.com/projectdiscovery/fastdialer v0.1.0/go.mod h1:jKOWzyHx+Q2sMiYst5HP6tWLQLDFPVgWBR72szn2h8w=
github.com/projectdiscovery/fasttemplate v0.0.2 h1:h2cISk5xDhlJEinlBQS6RRx0vOlOirB2y3Yu4PJzpiA=
From f43588dcd287245133b7ed62f5b32bed1b10746f Mon Sep 17 00:00:00 2001
From: sandeep <8293321+ehsandeep@users.noreply.github.com>
Date: Sat, 25 May 2024 02:52:49 +0530
Subject: [PATCH 023/283] version update
---
pkg/catalog/config/constants.go | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/pkg/catalog/config/constants.go b/pkg/catalog/config/constants.go
index 3918a0124a..bfd95f1b88 100644
--- a/pkg/catalog/config/constants.go
+++ b/pkg/catalog/config/constants.go
@@ -31,7 +31,7 @@ const (
CLIConfigFileName = "config.yaml"
ReportingConfigFilename = "reporting-config.yaml"
// Version is the current version of nuclei
- Version = `v3.2.8-dev`
+ Version = `v3.2.8`
// Directory Names of custom templates
CustomS3TemplatesDirName = "s3"
CustomGitHubTemplatesDirName = "github"
From 718aca6e0e7e612a263f1a0965a631149914dd99 Mon Sep 17 00:00:00 2001
From: sandeep <8293321+ehsandeep@users.noreply.github.com>
Date: Sat, 25 May 2024 04:03:25 +0530
Subject: [PATCH 024/283] go mod update
---
go.mod | 12 ++++++------
go.sum | 22 ++++++++++++----------
2 files changed, 18 insertions(+), 16 deletions(-)
diff --git a/go.mod b/go.mod
index 8368b1afd3..9006cea29b 100644
--- a/go.mod
+++ b/go.mod
@@ -39,7 +39,7 @@ require (
github.com/xanzy/go-gitlab v0.84.0
go.uber.org/multierr v1.11.0
golang.org/x/net v0.25.0
- golang.org/x/oauth2 v0.11.0
+ golang.org/x/oauth2 v0.18.0
golang.org/x/text v0.15.0
gopkg.in/yaml.v2 v2.4.0
)
@@ -81,7 +81,7 @@ require (
github.com/projectdiscovery/dsl v0.0.57
github.com/projectdiscovery/fasttemplate v0.0.2
github.com/projectdiscovery/go-smb2 v0.0.0-20240129202741-052cc450c6cb
- github.com/projectdiscovery/goflags v0.1.52
+ github.com/projectdiscovery/goflags v0.1.53
github.com/projectdiscovery/gologger v1.1.12
github.com/projectdiscovery/gostruct v0.0.2
github.com/projectdiscovery/gozero v0.0.2
@@ -92,7 +92,7 @@ require (
github.com/projectdiscovery/rdap v0.9.1-0.20221108103045-9865884d1917
github.com/projectdiscovery/sarif v0.0.1
github.com/projectdiscovery/tlsx v1.1.6
- github.com/projectdiscovery/uncover v1.0.7
+ github.com/projectdiscovery/uncover v1.0.8
github.com/projectdiscovery/useragent v0.0.52
github.com/projectdiscovery/utils v0.1.0
github.com/projectdiscovery/wappalyzergo v0.1.0
@@ -260,7 +260,7 @@ require (
github.com/gobwas/pool v0.2.1 // indirect
github.com/golang-jwt/jwt/v4 v4.5.0 // indirect
github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da // indirect
- github.com/golang/protobuf v1.5.3 // indirect
+ github.com/golang/protobuf v1.5.4 // indirect
github.com/golang/snappy v0.0.4 // indirect
github.com/google/go-querystring v1.1.0 // indirect
github.com/google/uuid v1.3.1 // indirect
@@ -289,7 +289,7 @@ require (
github.com/tklauser/go-sysconf v0.3.12 // indirect
github.com/tklauser/numcpus v0.6.1 // indirect
github.com/trivago/tgo v1.0.7
- github.com/ulikunitz/xz v0.5.11 // indirect
+ github.com/ulikunitz/xz v0.5.12 // indirect
github.com/valyala/bytebufferpool v1.0.0 // indirect
github.com/yl2chen/cidranger v1.0.2 // indirect
github.com/ysmood/goob v0.4.0 // indirect
@@ -307,7 +307,7 @@ require (
golang.org/x/sys v0.20.0 // indirect
golang.org/x/time v0.5.0 // indirect
golang.org/x/tools v0.21.0
- google.golang.org/appengine v1.6.7 // indirect
+ google.golang.org/appengine v1.6.8 // indirect
google.golang.org/protobuf v1.33.0 // indirect
gopkg.in/alecthomas/kingpin.v2 v2.2.6 // indirect
gopkg.in/corvus-ch/zbase32.v1 v1.0.0 // indirect
diff --git a/go.sum b/go.sum
index 1c56ada6a6..a65794fa25 100644
--- a/go.sum
+++ b/go.sum
@@ -450,8 +450,9 @@ github.com/golang/protobuf v1.4.2/go.mod h1:oDoupMAO8OvCJWAcko0GGGIgR6R6ocIYbsSw
github.com/golang/protobuf v1.4.3/go.mod h1:oDoupMAO8OvCJWAcko0GGGIgR6R6ocIYbsSw735rRwI=
github.com/golang/protobuf v1.5.0/go.mod h1:FsONVRAS9T7sI+LIUmWTfcYkHO4aIWwzhcaSAoJOfIk=
github.com/golang/protobuf v1.5.2/go.mod h1:XVQd3VNwM+JqD3oG2Ue2ip4fOMUkwXdXDdiuN0vRsmY=
-github.com/golang/protobuf v1.5.3 h1:KhyjKVUg7Usr/dYsdSqoFveMYd5ko72D+zANwlG1mmg=
github.com/golang/protobuf v1.5.3/go.mod h1:XVQd3VNwM+JqD3oG2Ue2ip4fOMUkwXdXDdiuN0vRsmY=
+github.com/golang/protobuf v1.5.4 h1:i7eJL8qZTpSEXOPTxNKhASYpMn+8e5Q6AdndVa1dWek=
+github.com/golang/protobuf v1.5.4/go.mod h1:lnTiLA8Wa4RWRcIUkrtSVa5nRhsEGBg48fD6rSs7xps=
github.com/golang/snappy v0.0.0-20180518054509-2e65f85255db/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q=
github.com/golang/snappy v0.0.2/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q=
github.com/golang/snappy v0.0.4 h1:yAGX7huGHXlcLOEtBnF4w7FQwA26wojNCwOYAEhLjQM=
@@ -839,8 +840,8 @@ github.com/projectdiscovery/freeport v0.0.5 h1:jnd3Oqsl4S8n0KuFkE5Hm8WGDP24ITBvm
github.com/projectdiscovery/freeport v0.0.5/go.mod h1:PY0bxSJ34HVy67LHIeF3uIutiCSDwOqKD8ruBkdiCwE=
github.com/projectdiscovery/go-smb2 v0.0.0-20240129202741-052cc450c6cb h1:rutG906Drtbpz4DwU5mhGIeOhRcktDH4cGQitGUMAsg=
github.com/projectdiscovery/go-smb2 v0.0.0-20240129202741-052cc450c6cb/go.mod h1:FLjF1DmZ+POoGEiIQdWuYVwS++C/GwpX8YaCsTSm1RY=
-github.com/projectdiscovery/goflags v0.1.52 h1:rVYZOtq7iA8e6ceyVZbp6OcuMhcwh5weiXSuDoXsivU=
-github.com/projectdiscovery/goflags v0.1.52/go.mod h1:tcBQ0EVGP4Wafza7gx57ZktkGxyfdLn+eQWUUQrV84c=
+github.com/projectdiscovery/goflags v0.1.53 h1:wXNIMjvjBo0Hvo1183DQE2RyqlEbrhink/jYrFwRMwE=
+github.com/projectdiscovery/goflags v0.1.53/go.mod h1:yw4HnwPGKeCHx48tcl9ZqLrKeSArtUv2LK1dJP1i380=
github.com/projectdiscovery/gologger v1.1.12 h1:uX/QkQdip4PubJjjG0+uk5DtyAi1ANPJUvpmimXqv4A=
github.com/projectdiscovery/gologger v1.1.12/go.mod h1:DI8nywPLERS5mo8QEA9E7gd5HZ3Je14SjJBH3F5/kLw=
github.com/projectdiscovery/gostruct v0.0.2 h1:s8gP8ApugGM4go1pA+sVlPDXaWqNP5BBDDSv7VEdG1M=
@@ -879,8 +880,8 @@ github.com/projectdiscovery/stringsutil v0.0.2 h1:uzmw3IVLJSMW1kEg8eCStG/cGbYYZA
github.com/projectdiscovery/stringsutil v0.0.2/go.mod h1:EJ3w6bC5fBYjVou6ryzodQq37D5c6qbAYQpGmAy+DC0=
github.com/projectdiscovery/tlsx v1.1.6 h1:iw2zwKbd2+kRQ8J1G4dLmS0CLyemd/tKz1UzcNsC77A=
github.com/projectdiscovery/tlsx v1.1.6/go.mod h1:s7SRRFdrwIZBK/RXXZi4CR/CubqFSvp8h5Bk1srEZIo=
-github.com/projectdiscovery/uncover v1.0.7 h1:ut+2lTuvmftmveqF5RTjMWAgyLj8ltPQC7siFy9sj0A=
-github.com/projectdiscovery/uncover v1.0.7/go.mod h1:HFXgm1sRPuoN0D4oATljPIdmbo/EEh1wVuxQqo/dwFE=
+github.com/projectdiscovery/uncover v1.0.8 h1:nE8bYJuwhqk0BEMRQRhNbjPGMy40A9gkiMk5xq6U4u0=
+github.com/projectdiscovery/uncover v1.0.8/go.mod h1:1SwsNqjwMkJEzJQ7lQr5AHIdYd+BQlyqhO5IwIYmIAM=
github.com/projectdiscovery/useragent v0.0.52 h1:9SUPH0Epo3DJfB6PCDgETfMaD6nZ08sFvfgXTmPUAsU=
github.com/projectdiscovery/useragent v0.0.52/go.mod h1:PUXHgShvaD8p3bihy1mY8tuBDhdk3M0yy4Z10Ajg2yQ=
github.com/projectdiscovery/utils v0.1.0 h1:r7Z/s2CBktJ0bnSN410lzOhD8S/0IxmzmFxkQudYKps=
@@ -1048,8 +1049,8 @@ github.com/ugorji/go/codec v1.2.11 h1:BMaWp1Bb6fHwEtbplGBGJ498wD+LKlNSl25MjdZY4d
github.com/ugorji/go/codec v1.2.11/go.mod h1:UNopzCgEMSXjBc6AOMqYvWC1ktqTAfzJZUZgYf6w6lg=
github.com/ulikunitz/xz v0.5.8/go.mod h1:nbz6k7qbPmH4IRqmfOplQw/tblSgqTqBwxkY0oWt/14=
github.com/ulikunitz/xz v0.5.9/go.mod h1:nbz6k7qbPmH4IRqmfOplQw/tblSgqTqBwxkY0oWt/14=
-github.com/ulikunitz/xz v0.5.11 h1:kpFauv27b6ynzBNT/Xy+1k+fK4WswhN/6PN5WhFAGw8=
-github.com/ulikunitz/xz v0.5.11/go.mod h1:nbz6k7qbPmH4IRqmfOplQw/tblSgqTqBwxkY0oWt/14=
+github.com/ulikunitz/xz v0.5.12 h1:37Nm15o69RwBkXM0J6A5OlE67RZTfzUxTj8fB3dfcsc=
+github.com/ulikunitz/xz v0.5.12/go.mod h1:nbz6k7qbPmH4IRqmfOplQw/tblSgqTqBwxkY0oWt/14=
github.com/urfave/cli v1.20.0/go.mod h1:70zkFmudgCuE/ngEzBv17Jvp/497gISqfk5gWijbERA=
github.com/urfave/cli v1.22.1/go.mod h1:Gos4lmkARVdJ6EkW0WaNv/tZAAMe9V7XWyB60NtXRu0=
github.com/valyala/bytebufferpool v1.0.0 h1:GqA5TC/0021Y/b9FG4Oi9Mr3q7XYx6KllzawFIhcdPw=
@@ -1288,8 +1289,8 @@ golang.org/x/oauth2 v0.0.0-20210514164344-f6687ab2804c/go.mod h1:KelEdhl1UZF7XfJ
golang.org/x/oauth2 v0.0.0-20220223155221-ee480838109b/go.mod h1:DAh4E804XQdzx2j+YRIaUnCqCV2RuMz24cGBJ5QYIrc=
golang.org/x/oauth2 v0.5.0/go.mod h1:9/XBHVqLaWO3/BRHs5jbpYCnOZVjj5V0ndyaAM7KB4I=
golang.org/x/oauth2 v0.6.0/go.mod h1:ycmewcwgD4Rpr3eZJLSB4Kyyljb3qDh40vJ8STE5HKw=
-golang.org/x/oauth2 v0.11.0 h1:vPL4xzxBM4niKCW6g9whtaWVXTJf1U5e4aZxxFx/gbU=
-golang.org/x/oauth2 v0.11.0/go.mod h1:LdF7O/8bLR/qWK9DrpXmbHLTouvRHK0SgJl0GmDBchk=
+golang.org/x/oauth2 v0.18.0 h1:09qnuIAgzdx1XplqJvW6CQqMCtGZykZWcXzPMPUusvI=
+golang.org/x/oauth2 v0.18.0/go.mod h1:Wf7knwG0MPoWIMMBgFlEaSUDaKskp0dCfrlJRJXbBi8=
golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
golang.org/x/sync v0.0.0-20181221193216-37e7f081c4d4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
@@ -1498,8 +1499,9 @@ google.golang.org/appengine v1.5.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7
google.golang.org/appengine v1.6.1/go.mod h1:i06prIuMbXzDqacNJfV5OdTW448YApPu5ww/cMBSeb0=
google.golang.org/appengine v1.6.5/go.mod h1:8WjMMxjGQR8xUklV/ARdw2HLXBOI7O7uCIDZVag1xfc=
google.golang.org/appengine v1.6.6/go.mod h1:8WjMMxjGQR8xUklV/ARdw2HLXBOI7O7uCIDZVag1xfc=
-google.golang.org/appengine v1.6.7 h1:FZR1q0exgwxzPzp/aF+VccGrSfxfPpkBqjIIEq3ru6c=
google.golang.org/appengine v1.6.7/go.mod h1:8WjMMxjGQR8xUklV/ARdw2HLXBOI7O7uCIDZVag1xfc=
+google.golang.org/appengine v1.6.8 h1:IhEN5q69dyKagZPYMSdIjS2HqprW324FRQZJcGqPAsM=
+google.golang.org/appengine v1.6.8/go.mod h1:1jJ3jBArFh5pcgW8gCtRJnepW8FzD1V44FJffLiz/Ds=
google.golang.org/genproto v0.0.0-20180817151627-c66870c02cf8/go.mod h1:JiN7NxoALGmiZfu7CAH4rXhgtRTLTxftemlI0sWmxmc=
google.golang.org/genproto v0.0.0-20190307195333-5fe7a883aa19/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE=
google.golang.org/genproto v0.0.0-20190418145605-e7d98fc518a7/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE=
From 3307517a6cef118d705fa27ccfae1246d50db941 Mon Sep 17 00:00:00 2001
From: mzack
Date: Sat, 25 May 2024 02:52:55 +0200
Subject: [PATCH 025/283] conflict
---
lib/sdk_private.go | 1 +
1 file changed, 1 insertion(+)
diff --git a/lib/sdk_private.go b/lib/sdk_private.go
index c620340f37..0b05059d06 100644
--- a/lib/sdk_private.go
+++ b/lib/sdk_private.go
@@ -25,6 +25,7 @@ import (
"github.com/projectdiscovery/nuclei/v3/pkg/protocols/common/hosterrorscache"
"github.com/projectdiscovery/nuclei/v3/pkg/protocols/common/interactsh"
"github.com/projectdiscovery/nuclei/v3/pkg/protocols/common/protocolinit"
+ "github.com/projectdiscovery/nuclei/v3/pkg/protocols/common/protocolstate"
"github.com/projectdiscovery/nuclei/v3/pkg/protocols/http/httpclientpool"
"github.com/projectdiscovery/nuclei/v3/pkg/reporting"
"github.com/projectdiscovery/nuclei/v3/pkg/templates"
From 281b24ed50c9945c3c3ba7f9125e08d98708ace5 Mon Sep 17 00:00:00 2001
From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com>
Date: Mon, 27 May 2024 05:49:01 +0000
Subject: [PATCH 026/283] chore(deps): bump github.com/projectdiscovery/hmap
from 0.0.43 to 0.0.44
Bumps [github.com/projectdiscovery/hmap](https://github.com/projectdiscovery/hmap) from 0.0.43 to 0.0.44.
- [Release notes](https://github.com/projectdiscovery/hmap/releases)
- [Commits](https://github.com/projectdiscovery/hmap/compare/v0.0.43...v0.0.44)
---
updated-dependencies:
- dependency-name: github.com/projectdiscovery/hmap
dependency-type: direct:production
update-type: version-update:semver-patch
...
Signed-off-by: dependabot[bot]
---
go.mod | 2 +-
go.sum | 4 ++--
2 files changed, 3 insertions(+), 3 deletions(-)
diff --git a/go.mod b/go.mod
index 9006cea29b..330ccf252b 100644
--- a/go.mod
+++ b/go.mod
@@ -21,7 +21,7 @@ require (
github.com/pkg/errors v0.9.1
github.com/projectdiscovery/clistats v0.0.20
github.com/projectdiscovery/fastdialer v0.1.0
- github.com/projectdiscovery/hmap v0.0.43
+ github.com/projectdiscovery/hmap v0.0.44
github.com/projectdiscovery/interactsh v1.1.9
github.com/projectdiscovery/rawhttp v0.1.49
github.com/projectdiscovery/retryabledns v1.0.60
diff --git a/go.sum b/go.sum
index a65794fa25..452255cb71 100644
--- a/go.sum
+++ b/go.sum
@@ -848,8 +848,8 @@ github.com/projectdiscovery/gostruct v0.0.2 h1:s8gP8ApugGM4go1pA+sVlPDXaWqNP5BBD
github.com/projectdiscovery/gostruct v0.0.2/go.mod h1:H86peL4HKwMXcQQtEa6lmC8FuD9XFt6gkNR0B/Mu5PE=
github.com/projectdiscovery/gozero v0.0.2 h1:8fJeaCjxL9tpm33uG/RsCQs6HGM/NE6eA3cjkilRQ+E=
github.com/projectdiscovery/gozero v0.0.2/go.mod h1:d8bZvDWW07LWNYWrwjZ4OO1I0cpkfqaysyDfSs9ibK8=
-github.com/projectdiscovery/hmap v0.0.43 h1:9A/lGm/0uP9x2hP90dmK5vkc6KfGD0XYxDBAQ7lx/zg=
-github.com/projectdiscovery/hmap v0.0.43/go.mod h1:cMLXGlkLcPK0Y/aJVa+IZM9wdtkQ6HX4mHTXVC05Jbc=
+github.com/projectdiscovery/hmap v0.0.44 h1:bFUmPltPPZH+oGDnbYLEXNujZe+0j81c1E+gF1MMLXQ=
+github.com/projectdiscovery/hmap v0.0.44/go.mod h1:glLxv5kLMj/Rgn6iZ9JiSzs72t8FQ4hA34zVoFwFhuM=
github.com/projectdiscovery/httpx v1.6.1 h1:aqKzoSydzKAJBV3Sxbqt44nTah4fNS9GkSD4Dy6Fzlc=
github.com/projectdiscovery/httpx v1.6.1/go.mod h1:RvibGQbm2Bh2DtuBDVXHLfauFDLtp7opKHyk65sz0ss=
github.com/projectdiscovery/interactsh v1.1.9 h1:b77SaSGrO+DtivmWwqGGY2dmNlQC3Zgmwlaj9L4Oqvc=
From 527f18a6e650f86bed0cad8be1e8d72c82d537cc Mon Sep 17 00:00:00 2001
From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com>
Date: Mon, 27 May 2024 05:49:07 +0000
Subject: [PATCH 027/283] chore(deps): bump
github.com/projectdiscovery/ratelimit
Bumps [github.com/projectdiscovery/ratelimit](https://github.com/projectdiscovery/ratelimit) from 0.0.42 to 0.0.43.
- [Release notes](https://github.com/projectdiscovery/ratelimit/releases)
- [Commits](https://github.com/projectdiscovery/ratelimit/compare/v0.0.42...v0.0.43)
---
updated-dependencies:
- dependency-name: github.com/projectdiscovery/ratelimit
dependency-type: direct:production
update-type: version-update:semver-patch
...
Signed-off-by: dependabot[bot]
---
go.mod | 2 +-
go.sum | 4 ++--
2 files changed, 3 insertions(+), 3 deletions(-)
diff --git a/go.mod b/go.mod
index 9006cea29b..fe9aa4f2db 100644
--- a/go.mod
+++ b/go.mod
@@ -88,7 +88,7 @@ require (
github.com/projectdiscovery/httpx v1.6.1
github.com/projectdiscovery/mapcidr v1.1.34
github.com/projectdiscovery/n3iwf v0.0.0-20230523120440-b8cd232ff1f5
- github.com/projectdiscovery/ratelimit v0.0.42
+ github.com/projectdiscovery/ratelimit v0.0.43
github.com/projectdiscovery/rdap v0.9.1-0.20221108103045-9865884d1917
github.com/projectdiscovery/sarif v0.0.1
github.com/projectdiscovery/tlsx v1.1.6
diff --git a/go.sum b/go.sum
index a65794fa25..e0f01ed693 100644
--- a/go.sum
+++ b/go.sum
@@ -864,8 +864,8 @@ github.com/projectdiscovery/n3iwf v0.0.0-20230523120440-b8cd232ff1f5 h1:L/e8z8yw
github.com/projectdiscovery/n3iwf v0.0.0-20230523120440-b8cd232ff1f5/go.mod h1:pGW2ncnTxTxHtP9wzcIJAB+3/NMp6IiuQWd2NK7K+oc=
github.com/projectdiscovery/networkpolicy v0.0.8 h1:XvfBaBwSDNTesSfNQP9VLk3HX9I7x7gHm028TJ5XwI8=
github.com/projectdiscovery/networkpolicy v0.0.8/go.mod h1:xnjNqhemxUPxU+UD5Jgsc3+K8IVmcqT1SJeo6UzMtkI=
-github.com/projectdiscovery/ratelimit v0.0.42 h1:110tBLaGUgyPK0DjLmMBQaaqW4tkmqsFidr/t6tyOy4=
-github.com/projectdiscovery/ratelimit v0.0.42/go.mod h1:ruhLiZ5liukSpG07p6eHTCPJUmTwOhxDSxQPulvC3/Y=
+github.com/projectdiscovery/ratelimit v0.0.43 h1:9yYCnF+NW/vIRVmyvmyCsIph9Z3nsCDka2BrRciDg4c=
+github.com/projectdiscovery/ratelimit v0.0.43/go.mod h1:pWKRf/OY4uKtjelBLS5G3mclgvlwpBtbjcYpxdLQsL0=
github.com/projectdiscovery/rawhttp v0.1.49 h1:OPP9R/UZx/GFrcPRUs9fOS1dLwhg+2o7p3dByzkIhWM=
github.com/projectdiscovery/rawhttp v0.1.49/go.mod h1:aaAaMsdzHPfw4yU3nbeP7NI3vy1ZjgoXw7l+m4Tnt94=
github.com/projectdiscovery/rdap v0.9.1-0.20221108103045-9865884d1917 h1:m03X4gBVSorSzvmm0bFa7gDV4QNSOWPL/fgZ4kTXBxk=
From 75d0af1aea65026a93afe2bc4546b4942323a3ec Mon Sep 17 00:00:00 2001
From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com>
Date: Mon, 27 May 2024 05:49:18 +0000
Subject: [PATCH 028/283] chore(deps): bump github.com/projectdiscovery/goflags
Bumps [github.com/projectdiscovery/goflags](https://github.com/projectdiscovery/goflags) from 0.1.53 to 0.1.54.
- [Release notes](https://github.com/projectdiscovery/goflags/releases)
- [Commits](https://github.com/projectdiscovery/goflags/compare/v0.1.53...v0.1.54)
---
updated-dependencies:
- dependency-name: github.com/projectdiscovery/goflags
dependency-type: direct:production
update-type: version-update:semver-patch
...
Signed-off-by: dependabot[bot]
---
go.mod | 2 +-
go.sum | 4 ++--
2 files changed, 3 insertions(+), 3 deletions(-)
diff --git a/go.mod b/go.mod
index 9006cea29b..32f61a0097 100644
--- a/go.mod
+++ b/go.mod
@@ -81,7 +81,7 @@ require (
github.com/projectdiscovery/dsl v0.0.57
github.com/projectdiscovery/fasttemplate v0.0.2
github.com/projectdiscovery/go-smb2 v0.0.0-20240129202741-052cc450c6cb
- github.com/projectdiscovery/goflags v0.1.53
+ github.com/projectdiscovery/goflags v0.1.54
github.com/projectdiscovery/gologger v1.1.12
github.com/projectdiscovery/gostruct v0.0.2
github.com/projectdiscovery/gozero v0.0.2
diff --git a/go.sum b/go.sum
index a65794fa25..eccf11d2b3 100644
--- a/go.sum
+++ b/go.sum
@@ -840,8 +840,8 @@ github.com/projectdiscovery/freeport v0.0.5 h1:jnd3Oqsl4S8n0KuFkE5Hm8WGDP24ITBvm
github.com/projectdiscovery/freeport v0.0.5/go.mod h1:PY0bxSJ34HVy67LHIeF3uIutiCSDwOqKD8ruBkdiCwE=
github.com/projectdiscovery/go-smb2 v0.0.0-20240129202741-052cc450c6cb h1:rutG906Drtbpz4DwU5mhGIeOhRcktDH4cGQitGUMAsg=
github.com/projectdiscovery/go-smb2 v0.0.0-20240129202741-052cc450c6cb/go.mod h1:FLjF1DmZ+POoGEiIQdWuYVwS++C/GwpX8YaCsTSm1RY=
-github.com/projectdiscovery/goflags v0.1.53 h1:wXNIMjvjBo0Hvo1183DQE2RyqlEbrhink/jYrFwRMwE=
-github.com/projectdiscovery/goflags v0.1.53/go.mod h1:yw4HnwPGKeCHx48tcl9ZqLrKeSArtUv2LK1dJP1i380=
+github.com/projectdiscovery/goflags v0.1.54 h1:xbaEBNbKqXyRu4154UlhsMAFmpaDYma4jJ9zUZGvXW8=
+github.com/projectdiscovery/goflags v0.1.54/go.mod h1:cfLzWWxgl/ft5cSHVJZjnvIzd4wX3A9Kz+W1pjeBZvc=
github.com/projectdiscovery/gologger v1.1.12 h1:uX/QkQdip4PubJjjG0+uk5DtyAi1ANPJUvpmimXqv4A=
github.com/projectdiscovery/gologger v1.1.12/go.mod h1:DI8nywPLERS5mo8QEA9E7gd5HZ3Je14SjJBH3F5/kLw=
github.com/projectdiscovery/gostruct v0.0.2 h1:s8gP8ApugGM4go1pA+sVlPDXaWqNP5BBDDSv7VEdG1M=
From a2a85fb00271c97e43d2faded6c4c38e2bd156d2 Mon Sep 17 00:00:00 2001
From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com>
Date: Mon, 27 May 2024 06:20:04 +0000
Subject: [PATCH 029/283] chore(deps): bump github.com/projectdiscovery/rawhttp
Bumps [github.com/projectdiscovery/rawhttp](https://github.com/projectdiscovery/rawhttp) from 0.1.49 to 0.1.51.
- [Release notes](https://github.com/projectdiscovery/rawhttp/releases)
- [Commits](https://github.com/projectdiscovery/rawhttp/compare/v0.1.49...v0.1.51)
---
updated-dependencies:
- dependency-name: github.com/projectdiscovery/rawhttp
dependency-type: direct:production
update-type: version-update:semver-patch
...
Signed-off-by: dependabot[bot]
---
go.mod | 2 +-
go.sum | 4 ++--
2 files changed, 3 insertions(+), 3 deletions(-)
diff --git a/go.mod b/go.mod
index 1014b3ae7d..f58131b9bb 100644
--- a/go.mod
+++ b/go.mod
@@ -23,7 +23,7 @@ require (
github.com/projectdiscovery/fastdialer v0.1.0
github.com/projectdiscovery/hmap v0.0.44
github.com/projectdiscovery/interactsh v1.1.9
- github.com/projectdiscovery/rawhttp v0.1.49
+ github.com/projectdiscovery/rawhttp v0.1.51
github.com/projectdiscovery/retryabledns v1.0.60
github.com/projectdiscovery/retryablehttp-go v1.0.62
github.com/projectdiscovery/yamldoc-go v1.0.4
diff --git a/go.sum b/go.sum
index 207ca8ec1a..3c61eba18d 100644
--- a/go.sum
+++ b/go.sum
@@ -866,8 +866,8 @@ github.com/projectdiscovery/networkpolicy v0.0.8 h1:XvfBaBwSDNTesSfNQP9VLk3HX9I7
github.com/projectdiscovery/networkpolicy v0.0.8/go.mod h1:xnjNqhemxUPxU+UD5Jgsc3+K8IVmcqT1SJeo6UzMtkI=
github.com/projectdiscovery/ratelimit v0.0.43 h1:9yYCnF+NW/vIRVmyvmyCsIph9Z3nsCDka2BrRciDg4c=
github.com/projectdiscovery/ratelimit v0.0.43/go.mod h1:pWKRf/OY4uKtjelBLS5G3mclgvlwpBtbjcYpxdLQsL0=
-github.com/projectdiscovery/rawhttp v0.1.49 h1:OPP9R/UZx/GFrcPRUs9fOS1dLwhg+2o7p3dByzkIhWM=
-github.com/projectdiscovery/rawhttp v0.1.49/go.mod h1:aaAaMsdzHPfw4yU3nbeP7NI3vy1ZjgoXw7l+m4Tnt94=
+github.com/projectdiscovery/rawhttp v0.1.51 h1:T4sskU2fsX9zNR7nyae0tD/0fliIs73KwBzyoJwBs10=
+github.com/projectdiscovery/rawhttp v0.1.51/go.mod h1:ejBaw3dCfJSfbtFv0ib6YfFSjQzt7ZdRVpQWJASxIUs=
github.com/projectdiscovery/rdap v0.9.1-0.20221108103045-9865884d1917 h1:m03X4gBVSorSzvmm0bFa7gDV4QNSOWPL/fgZ4kTXBxk=
github.com/projectdiscovery/rdap v0.9.1-0.20221108103045-9865884d1917/go.mod h1:JxXtZC9e195awe7EynrcnBJmFoad/BNDzW9mzFkK8Sg=
github.com/projectdiscovery/retryabledns v1.0.60 h1:f3TPeLnaJKrl9CbfsTfFTFnJY+UnyArH3mxxUUyR5ZQ=
From 8011012c420c8d3a56f3ee5812f472445e00669d Mon Sep 17 00:00:00 2001
From: Douglas Danger Manley
<127235272+doug-threatmate@users.noreply.github.com>
Date: Thu, 30 May 2024 06:34:15 -0400
Subject: [PATCH 030/283] Fix `ExecuteCallbackWithCtx` to use the context that
was provided (#5236)
* Fix `ExecuteCallbackWithCtx` to use the context that was provided
This updates `ExecuteCallbackWithCtx` to use the context that was
provided.
* remove more hardcoded context
---------
Co-authored-by: Tarun Koyalwar
---
lib/config.go | 8 +++++++-
lib/multi.go | 19 ++++++++++++-------
lib/sdk.go | 13 +++++++++----
lib/sdk_private.go | 12 ++++++------
lib/tests/sdk_test.go | 9 ++++++---
5 files changed, 40 insertions(+), 21 deletions(-)
diff --git a/lib/config.go b/lib/config.go
index 5610b71515..24bc08691f 100644
--- a/lib/config.go
+++ b/lib/config.go
@@ -164,11 +164,17 @@ func WithConcurrency(opts Concurrency) NucleiSDKOptions {
}
// WithGlobalRateLimit sets global rate (i.e all hosts combined) limit options
+// Deprecated: will be removed in favour of WithGlobalRateLimitCtx in next release
func WithGlobalRateLimit(maxTokens int, duration time.Duration) NucleiSDKOptions {
+ return WithGlobalRateLimitCtx(context.Background(), maxTokens, duration)
+}
+
+// WithGlobalRateLimitCtx allows setting a global rate limit for the entire engine
+func WithGlobalRateLimitCtx(ctx context.Context, maxTokens int, duration time.Duration) NucleiSDKOptions {
return func(e *NucleiEngine) error {
e.opts.RateLimit = maxTokens
e.opts.RateLimitDuration = duration
- e.rateLimiter = ratelimit.New(context.Background(), uint(e.opts.RateLimit), e.opts.RateLimitDuration)
+ e.rateLimiter = ratelimit.New(ctx, uint(e.opts.RateLimit), e.opts.RateLimitDuration)
return nil
}
}
diff --git a/lib/multi.go b/lib/multi.go
index 7c713e8aa0..a2149ddcd2 100644
--- a/lib/multi.go
+++ b/lib/multi.go
@@ -26,7 +26,7 @@ type unsafeOptions struct {
}
// createEphemeralObjects creates ephemeral nuclei objects/instances/types
-func createEphemeralObjects(base *NucleiEngine, opts *types.Options) (*unsafeOptions, error) {
+func createEphemeralObjects(ctx context.Context, base *NucleiEngine, opts *types.Options) (*unsafeOptions, error) {
u := &unsafeOptions{}
u.executerOpts = protocols.ExecutorOptions{
Output: base.customWriter,
@@ -49,9 +49,9 @@ func createEphemeralObjects(base *NucleiEngine, opts *types.Options) (*unsafeOpt
opts.RateLimitDuration = time.Second
}
if opts.RateLimit == 0 && opts.RateLimitDuration == 0 {
- u.executerOpts.RateLimiter = ratelimit.NewUnlimited(context.Background())
+ u.executerOpts.RateLimiter = ratelimit.NewUnlimited(ctx)
} else {
- u.executerOpts.RateLimiter = ratelimit.New(context.Background(), uint(opts.RateLimit), opts.RateLimitDuration)
+ u.executerOpts.RateLimiter = ratelimit.New(ctx, uint(opts.RateLimit), opts.RateLimitDuration)
}
u.engine = core.New(opts)
u.engine.SetExecuterOptions(u.executerOpts)
@@ -83,7 +83,7 @@ type ThreadSafeNucleiEngine struct {
// NewThreadSafeNucleiEngine creates a new nuclei engine with given options
// whose methods are thread-safe and can be used concurrently
// Note: Non-thread-safe methods start with Global prefix
-func NewThreadSafeNucleiEngine(opts ...NucleiSDKOptions) (*ThreadSafeNucleiEngine, error) {
+func NewThreadSafeNucleiEngineCtx(ctx context.Context, opts ...NucleiSDKOptions) (*ThreadSafeNucleiEngine, error) {
// default options
e := &NucleiEngine{
opts: types.DefaultOptions(),
@@ -94,12 +94,17 @@ func NewThreadSafeNucleiEngine(opts ...NucleiSDKOptions) (*ThreadSafeNucleiEngin
return nil, err
}
}
- if err := e.init(); err != nil {
+ if err := e.init(ctx); err != nil {
return nil, err
}
return &ThreadSafeNucleiEngine{eng: e}, nil
}
+// Deprecated: use NewThreadSafeNucleiEngineCtx instead
+func NewThreadSafeNucleiEngine(opts ...NucleiSDKOptions) (*ThreadSafeNucleiEngine, error) {
+ return NewThreadSafeNucleiEngineCtx(context.Background(), opts...)
+}
+
// GlobalLoadAllTemplates loads all templates from nuclei-templates repo
// This method will load all templates based on filters given at the time of nuclei engine creation in opts
func (e *ThreadSafeNucleiEngine) GlobalLoadAllTemplates() error {
@@ -124,7 +129,7 @@ func (e *ThreadSafeNucleiEngine) ExecuteNucleiWithOptsCtx(ctx context.Context, t
}
}
// create ephemeral nuclei objects/instances/types using base nuclei engine
- unsafeOpts, err := createEphemeralObjects(e.eng, tmpEngine.opts)
+ unsafeOpts, err := createEphemeralObjects(ctx, e.eng, tmpEngine.opts)
if err != nil {
return err
}
@@ -156,7 +161,7 @@ func (e *ThreadSafeNucleiEngine) ExecuteNucleiWithOptsCtx(ctx context.Context, t
engine := core.New(tmpEngine.opts)
engine.SetExecuterOptions(unsafeOpts.executerOpts)
- _ = engine.ExecuteScanWithOpts(context.Background(), store.Templates(), inputProvider, false)
+ _ = engine.ExecuteScanWithOpts(ctx, store.Templates(), inputProvider, false)
engine.WorkPool().Wait()
return nil
diff --git a/lib/sdk.go b/lib/sdk.go
index 63925f47ca..2e23aa49cd 100644
--- a/lib/sdk.go
+++ b/lib/sdk.go
@@ -240,7 +240,7 @@ func (e *NucleiEngine) ExecuteCallbackWithCtx(ctx context.Context, callback ...f
}
e.resultCallbacks = append(e.resultCallbacks, filtered...)
- _ = e.engine.ExecuteScanWithOpts(context.Background(), e.store.Templates(), e.inputProvider, false)
+ _ = e.engine.ExecuteScanWithOpts(ctx, e.store.Templates(), e.inputProvider, false)
defer e.engine.WorkPool().Wait()
return nil
}
@@ -261,8 +261,8 @@ func (e *NucleiEngine) Engine() *core.Engine {
return e.engine
}
-// NewNucleiEngine creates a new nuclei engine instance
-func NewNucleiEngine(options ...NucleiSDKOptions) (*NucleiEngine, error) {
+// NewNucleiEngineCtx creates a new nuclei engine instance with given context
+func NewNucleiEngineCtx(ctx context.Context, options ...NucleiSDKOptions) (*NucleiEngine, error) {
// default options
e := &NucleiEngine{
opts: types.DefaultOptions(),
@@ -273,8 +273,13 @@ func NewNucleiEngine(options ...NucleiSDKOptions) (*NucleiEngine, error) {
return nil, err
}
}
- if err := e.init(); err != nil {
+ if err := e.init(ctx); err != nil {
return nil, err
}
return e, nil
}
+
+// Deprecated: use NewNucleiEngineCtx instead
+func NewNucleiEngine(options ...NucleiSDKOptions) (*NucleiEngine, error) {
+ return NewNucleiEngineCtx(context.Background(), options...)
+}
diff --git a/lib/sdk_private.go b/lib/sdk_private.go
index cfe9c88a25..ae61add221 100644
--- a/lib/sdk_private.go
+++ b/lib/sdk_private.go
@@ -37,7 +37,7 @@ import (
var sharedInit sync.Once = sync.Once{}
// applyRequiredDefaults to options
-func (e *NucleiEngine) applyRequiredDefaults() {
+func (e *NucleiEngine) applyRequiredDefaults(ctx context.Context) {
mockoutput := testutils.NewMockOutputWriter(e.opts.OmitTemplate)
mockoutput.WriteCallback = func(event *output.ResultEvent) {
if len(e.resultCallbacks) > 0 {
@@ -81,7 +81,7 @@ func (e *NucleiEngine) applyRequiredDefaults() {
e.interactshOpts = interactsh.DefaultOptions(e.customWriter, e.rc, e.customProgress)
}
if e.rateLimiter == nil {
- e.rateLimiter = ratelimit.New(context.Background(), 150, time.Second)
+ e.rateLimiter = ratelimit.New(ctx, 150, time.Second)
}
if e.opts.ExcludeTags == nil {
e.opts.ExcludeTags = []string{}
@@ -94,7 +94,7 @@ func (e *NucleiEngine) applyRequiredDefaults() {
}
// init
-func (e *NucleiEngine) init() error {
+func (e *NucleiEngine) init(ctx context.Context) error {
if e.opts.Verbose {
gologger.DefaultLogger.SetMaxLevel(levels.LevelVerbose)
} else if e.opts.Debug {
@@ -121,7 +121,7 @@ func (e *NucleiEngine) init() error {
_ = protocolinit.Init(e.opts)
})
- e.applyRequiredDefaults()
+ e.applyRequiredDefaults(ctx)
var err error
// setup progressbar
@@ -204,9 +204,9 @@ func (e *NucleiEngine) init() error {
e.opts.RateLimitDuration = time.Second
}
if e.opts.RateLimit == 0 && e.opts.RateLimitDuration == 0 {
- e.executerOpts.RateLimiter = ratelimit.NewUnlimited(context.Background())
+ e.executerOpts.RateLimiter = ratelimit.NewUnlimited(ctx)
} else {
- e.executerOpts.RateLimiter = ratelimit.New(context.Background(), uint(e.opts.RateLimit), e.opts.RateLimitDuration)
+ e.executerOpts.RateLimiter = ratelimit.New(ctx, uint(e.opts.RateLimit), e.opts.RateLimitDuration)
}
}
diff --git a/lib/tests/sdk_test.go b/lib/tests/sdk_test.go
index b20c163d1a..97ec489abc 100644
--- a/lib/tests/sdk_test.go
+++ b/lib/tests/sdk_test.go
@@ -1,6 +1,7 @@
package sdk_test
import (
+ "context"
"os"
"os/exec"
"testing"
@@ -28,7 +29,8 @@ func TestSimpleNuclei(t *testing.T) {
time.Sleep(2 * time.Second)
goleak.VerifyNone(t, knownLeaks...)
}()
- ne, err := nuclei.NewNucleiEngine(
+ ne, err := nuclei.NewNucleiEngineCtx(
+ context.TODO(),
nuclei.WithTemplateFilters(nuclei.TemplateFilters{ProtocolTypes: "dns"}), // filter dns templates
nuclei.EnableStatsWithOpts(nuclei.StatsOptions{JSON: true}),
)
@@ -62,7 +64,8 @@ func TestSimpleNucleiRemote(t *testing.T) {
time.Sleep(2 * time.Second)
goleak.VerifyNone(t, knownLeaks...)
}()
- ne, err := nuclei.NewNucleiEngine(
+ ne, err := nuclei.NewNucleiEngineCtx(
+ context.TODO(),
nuclei.WithTemplatesOrWorkflows(
nuclei.TemplateSources{
RemoteTemplates: []string{"https://cloud.projectdiscovery.io/public/nameserver-fingerprint.yaml"},
@@ -100,7 +103,7 @@ func TestThreadSafeNuclei(t *testing.T) {
goleak.VerifyNone(t, knownLeaks...)
}()
// create nuclei engine with options
- ne, err := nuclei.NewThreadSafeNucleiEngine()
+ ne, err := nuclei.NewThreadSafeNucleiEngineCtx(context.TODO())
require.Nil(t, err)
// scan 1 = run dns templates on scanme.sh
From f8a4cf50c6eabd38d5536d5f0386e251c482d73f Mon Sep 17 00:00:00 2001
From: Jorge Machado
Date: Sun, 2 Jun 2024 08:41:21 -0300
Subject: [PATCH 031/283] Add Spanish translation of README (#5242)
---
README.md | 3 +-
README_CN.md | 3 +-
README_ES.md | 369 +++++++++++++++++++++++++++++++++++++++++++++++++++
README_ID.md | 3 +-
README_KR.md | 3 +-
5 files changed, 377 insertions(+), 4 deletions(-)
create mode 100644 README_ES.md
diff --git a/README.md b/README.md
index f898e326a4..661dcd626c 100644
--- a/README.md
+++ b/README.md
@@ -30,7 +30,8 @@
English •
中文 •
Korean •
- Indonesia
+ Indonesia •
+ Spanish
---
diff --git a/README_CN.md b/README_CN.md
index 25e39b2ae6..8dff0506c9 100644
--- a/README_CN.md
+++ b/README_CN.md
@@ -32,7 +32,8 @@
English •
中文 •
Korean •
- Indonesia
+ Indonesia •
+ Spanish
---
diff --git a/README_ES.md b/README_ES.md
new file mode 100644
index 0000000000..d908eb4c6a
--- /dev/null
+++ b/README_ES.md
@@ -0,0 +1,369 @@
+
+
+
+
+
+Escáner de vulnerabilidades rápido y personalizable basado en un sencillo DSL basado en YAML.
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ Cómo funciona •
+ Instalación •
+ Documentación •
+ Créditos •
+ Preguntas Frecuentes •
+ Únete a Discord
+
+
+
+ English •
+ 中文 •
+ Korean •
+ Indonesia •
+ Spanish
+
+
+---
+
+Nuclei se utiliza para enviar peticiones a múltiples objetivos basándose en una plantilla, lo que resulta en cero falsos positivos y proporciona un escaneo rápido en un gran número de hosts. Nuclei ofrece escaneos para una variedad de protocolos, incluyendo TCP, DNS, HTTP, SSL, File, Whois, Websocket, Headless, Code, etc. Con plantillas potentes y flexibles, Nuclei puede utilizarse para modelar todo tipo de comprobaciones de seguridad.
+
+Tenemos un [repositorio dedicado](https://github.com/projectdiscovery/nuclei-templates) que alberga varios tipos de plantillas de vulnerabilidades, contribuidas por **más de 300** investigadores y ingenieros de seguridad.
+
+## Cómo funciona
+
+
+
+
+
+
+
+| :exclamation: **Descargo de responsabilidad** |
+|---------------------------------|
+| **Este proyecto está en desarrollo activo**. Es de esperar que se produzcan cambios importantes con las nuevas versiones. Consulte el registro de cambios de la versión antes de actualizar. |
+| Este proyecto fue principalmente desarrollado para ser utilizado como una herramienta CLI independiente. **Ejecutar nuclei como un servicio puede suponer riesgos de seguridad.** Se recomienda utilizarlo con precaución y tomar medidas de seguridad adicionales. |
+
+# Instalación de Nuclei
+
+Nuclei requiere **go1.21** para instalarse correctamente. Ejecute el siguiente comando para instalar la última versión -
+
+```sh
+go install -v github.com/projectdiscovery/nuclei/v3/cmd/nuclei@latest
+```
+
+
+ Brew
+
+ ```sh
+ brew install nuclei
+ ```
+
+
+
+ Docker
+
+ ```sh
+ docker pull projectdiscovery/nuclei:latest
+ ```
+
+
+
+**Más métodos de instalación [pueden encontrarse aquí](https://docs.projectdiscovery.io/tools/nuclei/install).**
+
+
+
+
+
+### Plantillas de Nuclei
+
+Nuclei cuenta con soporte incorporado para la descarga/actualización automática de plantillas desde la versión [v2.5.2](https://github.com/projectdiscovery/nuclei/releases/tag/v2.5.2) en adelante. El proyecto [**Nuclei-Templates**](https://github.com/projectdiscovery/nuclei-templates) proporciona una lista de plantillas listas para usar, aportadas por la comunidad, y que se actualizan constantemente.
+
+También puedes utilizar la bandera `update-templates` para actualizar las plantillas de Nuclei en cualquier momento; puedes escribir tus propias pruebas para tu flujo de trabajo y necesidades individuales siguiendo la [guía de plantillas](https://docs.projectdiscovery.io/templates/) de Nuclei.
+
+La sintaxis de referencia YAML DSL está disponible [aquí](SYNTAX-REFERENCE.md).
+
+ |
+
+
+
+### Uso
+
+```sh
+nuclei -h
+```
+
+Esto mostrará ayuda sobre la herramienta. Aquí están todas las opciones que soporta.
+
+
+```console
+Nuclei es un escáner de vulnerabilidades rápido y basado en plantillas
+que se centra en su amplia configurabilidad, extensibilidad y facilidad de uso.
+
+Usage:
+ ./nuclei [flags]
+
+Flags:
+TARGET:
+ -u, -target string[] URLs/hosts a escanear
+ -l, -list string ruta al archivo que contiene la lista de URLs/hosts a escanear (uno por línea)
+ -eh, -exclude-hosts string[] hosts a excluir para escanear de la lista de entrada (ip, cidr, hostname)
+ -resume string reanudar el escaneo usando resume.cfg (la clusterización quedará inhabilitada)
+ -sa, -scan-all-ips escanear todas las IP asociadas al registro dns
+ -iv, -ip-version string[] versión IP a escanear del nombre de host (4,6) - (por defecto 4)
+
+TARGET-FORMAT:
+ -im, -input-mode string modo del archivo de entrada (list, burp, jsonl, yaml, openapi, swagger) (por defecto "list")
+ -ro, -required-only utilizar solo campos requeridos en el formato de entrada al generar peticiones
+ -sfv, -skip-format-validation saltar la validación de formato (como variables faltantes) al procesar el archivo de entrada
+
+TEMPLATES:
+ -nt, -new-templates ejecutar sólo las nuevas plantillas añadidas en la última versión de nuclei-templates
+ -ntv, -new-templates-version string[] ejecutar las nuevas plantillas añadidas en la versión especificada
+ -as, -automatic-scan escaneo web automático utilizando la detección de tecnología de wappalyzer para mapeo de etiquetas
+ -t, -templates string[] lista de plantillas o directorio de plantillas a ejecutar (separadas por comas, file)
+ -turl, -template-url string[] url de plantilla o lista que contiene urls de plantillas a ejecutar (separadas por comas, file)
+ -w, -workflows string[] lista de flujos de trabajo o directorio de flujos de trabajo a ejecutar (separadas por comas, file)
+ -wurl, -workflow-url string[] url de flujo de trabajo o lista que contiene urls de flujo de trabajo para ejecutar (separadas por comas, file)
+ -validate valida las plantillas pasadas a nuclei
+ -nss, -no-strict-syntax deshabilita la comprobación de sintaxis estricta en las plantillas
+ -td, -template-display muestra el contenido de las plantillas
+ -tl lista todas las plantillas disponibles
+ -tgl lista todas las etiquetas disponibles
+ -sign firma las plantillas con la clave privada definida en la variable de entorno NUCLEI_SIGNATURE_PRIVATE_KEY
+ -code habilita la carga de plantillas basadas en protocolos de código
+ -dut, -disable-unsigned-templates deshabilita la ejecución de plantillas no firmadas o plantillas con firma no coincidente
+
+FILTERING:
+ -a, -author string[] plantillas a ejecutar basadas en autores (separadas por comas, file)
+ -tags string[] plantillas a ejecutar basadas en etiquetas (separadas por comas, file)
+ -etags, -exclude-tags string[] plantillas a excluir basadas en etiquetas (separadas por comas, file)
+ -itags, -include-tags string[] etiquetas a ejecutar incluso si están excluidas ya sea por defecto o por configuración
+ -id, -template-id string[] plantillas a ejecutar basadas en IDs de plantilla (comma-separated, file, allow-wildcard)
+ -eid, -exclude-id string[] plantillas a excluir basadas en IDs de plantilla (separadas por comas, file)
+ -it, -include-templates string[] ruta al archivo de plantilla o directorio a ejecutar incluso si están excluidas ya sea por defecto o por configuración
+ -et, -exclude-templates string[] ruta al archivo de plantilla o directorio a excluir (separadas por comas, file)
+ -em, -exclude-matchers string[] matchers de plantilla a excluir en el resultado
+ -s, -severity value[] plantillas a ejecutar basadas en criticidad. Valores posibles: info, bajo, medio, alto, crítico, desconocido
+ -es, -exclude-severity value[] plantillas a excluir basadas en criticidad. Valores posibles: info, bajo, medio, alto, crítico, desconocido
+ -pt, -type value[] plantillas a ejecutar basadas en tipo de protocolo. Valores posibles: dns, file, http, headless, tcp, workflow, ssl, websocket, whois, code, javascript
+ -ept, -exclude-type value[] plantillas a excluir basadas en tipo de protocolo. Valores posibles: dns, file, http, headless, tcp, workflow, ssl, websocket, whois, code, javascript
+ -tc, -template-condition string[] plantillas a ejecutar basadas en condición de expresión
+
+OUTPUT:
+ -o, -output string archivo de salida donde guardar las incidencias/vulnerabilidades detectadas
+ -sresp, -store-resp almacenar todas las peticiones/respuestas enviadas por nuclei en el directorio de salida
+ -srd, -store-resp-dir string almacenar todas las peticiones/respuestas enviadas por nuclei en un directorio personalizado (por defecto "output")
+ -silent mostrar resultados únicamente
+ -nc, -no-color deshabilitar la coloración del contenido de salida (códigos de escape ANSI)
+ -j, -jsonl escribir la salida en formato JSONL(ines)
+ -irr, -include-rr -omit-raw incluir pares peticiones/respuesta en las salidas JSON, JSONL y Markdown (sólo para hallazgos) [OBSOLETO usar -omit-raw] (por defecto true)
+ -or, -omit-raw omitir los pares peticiones/respuesta en las salidas JSON, JSONL y Markdown (sólo para hallazgos)
+ -ot, -omit-template omitir plantilla codificada en la salida JSON, JSONL
+ -nm, -no-meta deshabilitar la impresión de metadatos de resultados en la salida cli
+ -ts, -timestamp habilitar la impresión de la marca de tiempo en la salida cli
+ -rdb, -report-db string base de datos de informes de nuclei (utilizarla siempre para persistir los datos de los informes)
+ -ms, -matcher-status mostrar el estado de fallo de coincidencia
+ -me, -markdown-export string directorio para exportar resultados en formato markdown
+ -se, -sarif-export string archivo para exportar resultados en formato SARIF
+ -je, -json-export string archivo para exportar resultados en formato JSON
+ -jle, -jsonl-export string archivo para exportar resultados en formato JSONL(ines)
+
+CONFIGURATIONS:
+ -config string ruta al archivo de configuración de nuclei
+ -fr, -follow-redirects habilitar el seguimiento de redirecciones para plantillas http
+ -fhr, -follow-host-redirects seguir redirecciones en el mismo host
+ -mr, -max-redirects int número máximo de redirecciones a seguir para plantillas http (por defecto 10)
+ -dr, -disable-redirects deshabilitar redirecciones para plantillas http
+ -rc, -report-config string archivo de configuración del módulo de informes de nuclei
+ -H, -header string[] encabezado/cookie personalizado a incluir en todas las peticiones http en formato header:value (cli, file)
+ -V, -var value variables personalizadas en formato key=value
+ -r, -resolvers string archivo que contiene lista de resolutores para nuclei
+ -sr, -system-resolvers utilizar resolución de DNS del sistema como fallback de error
+ -dc, -disable-clustering deshabilitar la clusterización de peticiones
+ -passive habilitar el modo de procesamiento pasivo de respuestas HTTP
+ -fh2, -force-http2 forzar la conexión http2 en las peticiones
+ -ev, -env-vars habilitar el uso de variables de entorno en la plantilla
+ -cc, -client-cert string archivo de certificado de cliente (codificado en PEM) utilizado para autenticarse contra los hosts escaneados
+ -ck, -client-key string archivo de clave de cliente (codificado en PEM) utilizado para autenticarse contra los hosts escaneados
+ -ca, -client-ca string archivo de autoridad de certificación de cliente (codificado en PEM) utilizado para autenticarse contra los hosts escaneados
+ -sml, -show-match-line mostrar líneas de coincidencia para plantillas de archivo, funciona solo con extractores
+ -ztls utilizar la biblioteca ztls con autofallback a estándar para tls13 [Obsoleto] autofallback a ztls está habilitado por defecto
+ -sni string nombre de host tls sni a usar (por defecto: nombre de dominio de entrada)
+ -dt, -dialer-timeout value tiempo de espera para peticiones de red
+ -dka, -dialer-keep-alive value duración de keep-alive para peticiones de red
+ -lfa, -allow-local-file-access permite el acceso a archivos (carga útil) en cualquier lugar del sistema
+ -lna, -restrict-local-network-access bloquea conexiones a la red local / privada
+ -i, -interface string interfaz de red a usar para el escaneo de red
+ -at, -attack-type string tipo de combinaciones de carga útil a realizar (batteringram, pitchfork, clusterbomb)
+ -sip, -source-ip string dirección ip de origen a usar para el escaneo de red
+ -rsr, -response-size-read int tamaño máximo de respuesta a leer en bytes (por defecto 10485760)
+ -rss, -response-size-save int tamaño máximo de respuesta a guardar en bytes (por defecto 1048576)
+ -reset reset elimina todos los archivos de configuración y datos de nuclei (incluidas las nuclei-templates)
+ -tlsi, -tls-impersonate habilitar client hello (ja3) tls randomization experimental
+
+INTERACTSH:
+ -iserver, -interactsh-server string url del servidor interactsh para instancia autoalojada (por defecto: oast.pro,oast.live,oast.site,oast.online,oast.fun,oast.me)
+ -itoken, -interactsh-token string token de autenticación del servidor interactsh autoalojado
+ -interactions-cache-size int número de peticiones a mantener en la caché de interacciones (por defecto 5000)
+ -interactions-eviction int número de segundos a esperar antes de eliminar las solicitudes de la caché (por defecto 60)
+ -interactions-poll-duration int número de segundos a esperar antes de cada solicitud de polling de interacciones (por defecto 5)
+ -interactions-cooldown-period int tiempo adicional para el polling de interacciones antes de salir (por defecto 5)
+ -ni, -no-interactsh desactivar el servidor interactsh para pruebas OAST, excluir plantillas basadas en OAST
+
+FUZZING:
+ -ft, -fuzzing-type string sobrescribe el tipo de fuzzing establecido en la plantilla (replace, prefix, postfix, infix)
+ -fm, -fuzzing-mode string sobrescribe el modo de fuzzing establecido en la plantilla (multiple, single)
+ -fuzz habilita la carga de plantillas de fuzzing (Obsoleto: usar -dast en su lugar)
+ -dast solo ejecuta plantillas DAST
+
+UNCOVER:
+ -uc, -uncover habilita el motor uncover
+ -uq, -uncover-query string[] consulta de búsqueda uncover
+ -ue, -uncover-engine string[] motor de búsqueda uncover (shodan,censys,fofa,shodan-idb,quake,hunter,zoomeye,netlas,criminalip,publicwww,hunterhow) (por defecto shodan)
+ -uf, -uncover-field string campos uncover a devolver (ip,port,host) (por defecto "ip:port")
+ -ul, -uncover-limit int resultados uncover a devolver (por defecto 100)
+ -ur, -uncover-ratelimit int sobrescribe el límite de velocidad de los motores con el límite de velocidad del motor uncover (por defecto 60 req/min) (por defecto 60)
+
+RATE-LIMIT:
+ -rl, -rate-limit int número máximo de peticiones a enviar por segundo (por defecto 150)
+ -rlm, -rate-limit-minute int número máximo de peticiones a enviar por minuto
+ -bs, -bulk-size int número máximo de hosts a ser analizados en paralelo por plantilla (por defecto 25)
+ -c, -concurrency int número máximo de plantillas a ejecutar en paralelo (por defecto 25)
+ -hbs, -headless-bulk-size int número máximo de hosts headless a ser analizados en paralelo por plantilla (por defecto 10)
+ -headc, -headless-concurrency int número máximo de plantillas headless a ejecutar en paralelo (por defecto 10)
+ -jsc, -js-concurrency int número máximo de entornos de ejecución de JavaScript a ejecutar en paralelo (por defecto 120)
+ -pc, -payload-concurrency int concurrencia máxima de carga útil para cada plantilla (por defecto 25)
+
+OPTIMIZATIONS:
+ -timeout int tiempo de espera en segundos (por defecto 10)
+ -retries int número de veces que se reintenta una petición fallida (por defecto 1)
+ -ldp, -leave-default-ports dejar puertos HTTP/HTTPS predeterminados (por ejemplo, host:80,host:443)
+ -mhe, -max-host-error int errores máximos para un host antes de omitirlo del escaneo (por defecto 30)
+ -te, -track-error string[] agrega el error dado a la lista de seguimiento de errores máximos por host (standard, file)
+ -nmhe, -no-mhe deshabilita la omisión del host del escaneo basado en errores
+ -project utiliza una carpeta de proyecto para evitar enviar la misma petición varias veces
+ -project-path string establece una ruta de proyecto específica (por defecto "/tmp")
+ -spm, -stop-at-first-match detiene el procesamiento de las peticiones HTTP después de la primera coincidencia (puede romper la lógica de la plantilla/flujo de trabajo)
+ -stream modo transmisión - comienza a trabajar sin ordenar la entrada
+ -ss, -scan-strategy value estrategia a utilizar mientras se escanea (auto/host-spray/template-spray) (por defecto auto)
+ -irt, -input-read-timeout value tiempo de espera en la lectura de entrada (por defecto 3m0s)
+ -nh, -no-httpx deshabilita análisis httpx para entradas que no son URL
+ -no-stdin deshabilita el procesamiento de la entrada estándar
+
+HEADLESS:
+ -headless habilita las plantillas que requieren soporte de navegadores sin interfaz gráfica (headless browser) (el usuario root en Linux deshabilitará el sandbox)
+ -page-timeout int segundos para esperar cada página en modo sin interfaz (por defecto 20)
+ -sb, -show-browser muestra el navegador en la pantalla al ejecutar plantillas con modo sin interfaz
+ -ho, -headless-options string[] inicia Chrome en modo sin interfaz con opciones adicionales
+ -sc, -system-chrome utiliza el navegador Chrome instalado localmente en lugar del instalado por nuclei
+ -lha, -list-headless-action lista de acciones sin interfaz disponibles
+
+DEBUG:
+ -debug muestra todas las peticiones y respuestas
+ -dreq, -debug-req muestra todas las peticiones enviadas
+ -dresp, -debug-resp muestra todas las respuestas recibidas
+ -p, -proxy string[] lista de proxies http/socks5 a utilizar (separados por comas o archivo de entrada)
+ -pi, -proxy-internal proxy para todas las peticiones internas
+ -ldf, -list-dsl-function lista todas las firmas de función DSL admitidas
+ -tlog, -trace-log string archivo a escribir el registro de traza de peticiones enviadas
+ -elog, -error-log string archivo a escribir el registro de error de peticiones enviadas
+ -version muestra la versión de nuclei
+ -hm, -hang-monitor habilita la monitorización de bloqueos de nuclei
+ -v, -verbose muestra salida detallada
+ -profile-mem string archivo opcional de volcado de memoria de nuclei
+ -vv muestra las plantillas cargadas para el escaneo
+ -svd, -show-var-dump muestra el volcado de variables para depuración
+ -ep, -enable-pprof habilita el servidor de depuración pprof
+ -tv, -templates-version muestra la versión de las plantillas nuclei (nuclei-templates) instaladas
+ -hc, -health-check ejecuta comprobación de diagnóstico
+
+UPDATE:
+ -up, -update actualiza el motor de nuclei a la última versión lanzada
+ -ut, -update-templates actualiza nuclei-templates a la última versión lanzada
+ -ud, -update-template-dir string directorio personalizado para instalar/actualizar nuclei-templates
+ -duc, -disable-update-check deshabilita la comprobación automática de actualizaciones de nuclei/templates
+
+STATISTICS:
+ -stats muestra estadísticas sobre el escaneo en ejecución
+ -sj, -stats-json muestra estadísticas en formato JSONL(ines)
+ -si, -stats-interval int número de segundos a esperar entre mostrar una actualización de estadísticas (por defecto 5)
+ -mp, -metrics-port int puerto para exponer métricas de nuclei (por defecto 9092)
+
+CLOUD:
+ -auth configura la clave de API del cloud de projectdiscovery (pdcp)
+ -cup, -cloud-upload sube los resultados del escaneo al dashboard de pdcp
+ -sid, -scan-id string sube los resultados del escaneo al ID de escaneo dado
+
+AUTHENTICATION:
+ -sf, -secret-file string[] ruta al archivo de configuración que contiene los secrets para el escaneo autenticado de nuclei
+ -ps, -prefetch-secrets precarga los secrets del archivo de secrets
+
+
+EXAMPLES:
+Ejecutar nuclei en un solo host:
+ $ nuclei -target example.com
+
+Ejecutar nuclei con directorios de plantillas específicos:
+ $ nuclei -target example.com -t http/cves/ -t ssl
+
+Ejecutar nuclei contra una lista de hosts:
+ $ nuclei -list hosts.txt
+
+Ejecutar nuclei con una salida JSON:
+ $ nuclei -target example.com -json-export output.json
+
+Ejecutar nuclei con salidas Markdown ordenadas (con variables de entorno):
+ $ MARKDOWN_EXPORT_SORT_MODE=template nuclei -target example.com -markdown-export nuclei_report/
+
+Documentación adicional disponible en: https://docs.nuclei.sh/getting-started/running
+```
+
+### Ejecutando Nuclei
+
+Consulta https://docs.projectdiscovery.io/tools/nuclei/running para obtener detalles sobre cómo ejecutar Nuclei.
+
+### Uso de Nuclei desde código Go
+
+La guía completa sobre cómo usar Nuclei como biblioteca/SDK está disponible en [godoc](https://pkg.go.dev/github.com/projectdiscovery/nuclei/v3/lib#section-readme).
+
+
+### Recursos
+
+Puedes acceder a la documentación principal de Nuclei en https://docs.projectdiscovery.io/tools/nuclei/, y obtener más información sobre Nuclei en la nube con [ProjectDiscovery Cloud Platform](https://cloud.projectdiscovery.io).
+
+¡Consulta https://docs.projectdiscovery.io/tools/nuclei/resources para obtener más recursos y videos sobre Nuclei!
+
+### Créditos
+
+Gracias a todos los increíbles [contribuyentes de la comunidad que enviaron PRs](https://github.com/projectdiscovery/nuclei/graphs/contributors) y mantienen este proyecto actualizado. :heart:
+
+Si tienes una idea o algún tipo de mejora, eres bienvenido a contribuir y participar en el Proyecto, siéntete libre de enviar tu PR.
+
+
+
+
+
+
+
+
+También echa un vistazo a los siguientes proyectos de código abierto similares que pueden adaptarse a tu flujo de trabajo:
+
+[FFuF](https://github.com/ffuf/ffuf), [Qsfuzz](https://github.com/ameenmaali/qsfuzz), [Inception](https://github.com/proabiral/inception), [Snallygaster](https://github.com/hannob/snallygaster), [Gofingerprint](https://github.com/Static-Flow/gofingerprint), [Sn1per](https://github.com/1N3/Sn1per/tree/master/templates), [Google tsunami](https://github.com/google/tsunami-security-scanner), [Jaeles](https://github.com/jaeles-project/jaeles), [ChopChop](https://github.com/michelin/ChopChop)
+
+### Licencia
+
+Nuclei se distribuye bajo la [Licencia MIT](https://github.com/projectdiscovery/nuclei/blob/main/LICENSE.md)
+
+
+
+
diff --git a/README_ID.md b/README_ID.md
index 73f0d86e8a..ff0621d5c5 100644
--- a/README_ID.md
+++ b/README_ID.md
@@ -32,7 +32,8 @@
English •
中文 •
Korean •
- Indonesia
+ Indonesia •
+ Spanish
---
diff --git a/README_KR.md b/README_KR.md
index db6820dd64..97abdc7752 100644
--- a/README_KR.md
+++ b/README_KR.md
@@ -30,7 +30,8 @@
English •
中文 •
- 한국어
+ 한국어 •
+ 스페인어
---
From 8720e4f8638b36bca7f7128a57c32abdbfd9b7ef Mon Sep 17 00:00:00 2001
From: Tarun Koyalwar <45962551+tarunKoyalwar@users.noreply.github.com>
Date: Sun, 2 Jun 2024 17:11:56 +0530
Subject: [PATCH 032/283] fix panic: ref #5217 (#5230)
---
pkg/protocols/http/httpclientpool/options.go | 23 +++++++++++++++++++-
1 file changed, 22 insertions(+), 1 deletion(-)
diff --git a/pkg/protocols/http/httpclientpool/options.go b/pkg/protocols/http/httpclientpool/options.go
index b6e60b6367..f24e421b86 100644
--- a/pkg/protocols/http/httpclientpool/options.go
+++ b/pkg/protocols/http/httpclientpool/options.go
@@ -7,6 +7,7 @@ import (
"time"
"github.com/projectdiscovery/rawhttp"
+ urlutil "github.com/projectdiscovery/utils/url"
)
// WithCustomTimeout is a configuration for custom timeout
@@ -39,7 +40,27 @@ func SendRawRequest(client *rawhttp.Client, opts *RawHttpRequestOpts) (*http.Res
// this error is caused when rawhttp client sends a corrupted or malformed request packet to server
// some servers may attempt gracefully shutdown but most will just abruptly close the connection which results
// in a connection reset by peer error and this can be safely assumed as 400 Bad Request in terms of normal http flow
- req, _ := http.NewRequest(opts.Method, opts.URL, opts.Body)
+ req, reqErr := http.NewRequest(opts.Method, opts.URL, opts.Body)
+ if reqErr != nil {
+ // failed to build new request mostly because of invalid url or body
+ // try again or else return urlErr
+ parsed, urlErr := urlutil.ParseAbsoluteURL(opts.URL, true)
+ if urlErr != nil {
+ return nil, err
+ }
+ req, reqErr = http.NewRequest(opts.Method, parsed.Host, opts.Body)
+ if reqErr != nil {
+ return nil, err
+ }
+ req.URL = parsed.URL
+ req.Header = opts.Headers
+ }
+
+ // if req is still nil, return error
+ if req == nil {
+ return nil, err
+ }
+
req.Header = opts.Headers
resp = &http.Response{
Request: req,
From 425b7d0cb5f3125cf2c898eda47b8f12e7fe4581 Mon Sep 17 00:00:00 2001
From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com>
Date: Mon, 3 Jun 2024 05:55:12 +0000
Subject: [PATCH 033/283] chore(deps): bump github.com/projectdiscovery/httpx
from 1.6.1 to 1.6.3
Bumps [github.com/projectdiscovery/httpx](https://github.com/projectdiscovery/httpx) from 1.6.1 to 1.6.3.
- [Release notes](https://github.com/projectdiscovery/httpx/releases)
- [Changelog](https://github.com/projectdiscovery/httpx/blob/main/.goreleaser.yml)
- [Commits](https://github.com/projectdiscovery/httpx/compare/v1.6.1...v1.6.3)
---
updated-dependencies:
- dependency-name: github.com/projectdiscovery/httpx
dependency-type: direct:production
update-type: version-update:semver-patch
...
Signed-off-by: dependabot[bot]
---
go.mod | 8 ++++----
go.sum | 16 ++++++++--------
2 files changed, 12 insertions(+), 12 deletions(-)
diff --git a/go.mod b/go.mod
index f58131b9bb..729913d2cc 100644
--- a/go.mod
+++ b/go.mod
@@ -78,14 +78,14 @@ require (
github.com/mholt/archiver v3.1.1+incompatible
github.com/ory/dockertest/v3 v3.10.0
github.com/praetorian-inc/fingerprintx v1.1.9
- github.com/projectdiscovery/dsl v0.0.57
+ github.com/projectdiscovery/dsl v0.1.0
github.com/projectdiscovery/fasttemplate v0.0.2
github.com/projectdiscovery/go-smb2 v0.0.0-20240129202741-052cc450c6cb
github.com/projectdiscovery/goflags v0.1.54
github.com/projectdiscovery/gologger v1.1.12
github.com/projectdiscovery/gostruct v0.0.2
github.com/projectdiscovery/gozero v0.0.2
- github.com/projectdiscovery/httpx v1.6.1
+ github.com/projectdiscovery/httpx v1.6.3
github.com/projectdiscovery/mapcidr v1.1.34
github.com/projectdiscovery/n3iwf v0.0.0-20230523120440-b8cd232ff1f5
github.com/projectdiscovery/ratelimit v0.0.43
@@ -93,9 +93,9 @@ require (
github.com/projectdiscovery/sarif v0.0.1
github.com/projectdiscovery/tlsx v1.1.6
github.com/projectdiscovery/uncover v1.0.8
- github.com/projectdiscovery/useragent v0.0.52
+ github.com/projectdiscovery/useragent v0.0.53
github.com/projectdiscovery/utils v0.1.0
- github.com/projectdiscovery/wappalyzergo v0.1.0
+ github.com/projectdiscovery/wappalyzergo v0.1.1
github.com/redis/go-redis/v9 v9.1.0
github.com/seh-msft/burpxml v1.0.1
github.com/stretchr/testify v1.9.0
diff --git a/go.sum b/go.sum
index 3c61eba18d..e2fbbeda7e 100644
--- a/go.sum
+++ b/go.sum
@@ -830,8 +830,8 @@ github.com/projectdiscovery/cdncheck v1.0.9 h1:BS15gzj9gb5AVSKqTDzPamfSgStu7nJQO
github.com/projectdiscovery/cdncheck v1.0.9/go.mod h1:18SSl1w7rMj53CGeRIZTbDoa286a6xZIxGbaiEo4Fxs=
github.com/projectdiscovery/clistats v0.0.20 h1:5jO5SLiRJ7f0nDV0ndBNmBeesbROouPooH+DGMgoWq4=
github.com/projectdiscovery/clistats v0.0.20/go.mod h1:GJ2av0KnOvK0AISQnP8hyDclYIji1LVkx2l0pwnzAu4=
-github.com/projectdiscovery/dsl v0.0.57 h1:BFjkhtJHzvxSNO6bJJ0UkA7tIwysgsyqWbuhn3FVQfI=
-github.com/projectdiscovery/dsl v0.0.57/go.mod h1:w9XL7AT/RDyg7yiFQN8eWWefXiYQt3mojPDdjyTdQ0k=
+github.com/projectdiscovery/dsl v0.1.0 h1:10xoL0V/btU646xH6ioGXKwrGt6grT+BIa6id1/4ifE=
+github.com/projectdiscovery/dsl v0.1.0/go.mod h1:JgOjLk54Tl9HG98N2SM5OaKKJ/STpI7yDiKW5rhV+yg=
github.com/projectdiscovery/fastdialer v0.1.0 h1:NiId7eXvYZjiBo83AnIZ6K8ZjzLJU8wGXdRLzuKcRrg=
github.com/projectdiscovery/fastdialer v0.1.0/go.mod h1:jKOWzyHx+Q2sMiYst5HP6tWLQLDFPVgWBR72szn2h8w=
github.com/projectdiscovery/fasttemplate v0.0.2 h1:h2cISk5xDhlJEinlBQS6RRx0vOlOirB2y3Yu4PJzpiA=
@@ -850,8 +850,8 @@ github.com/projectdiscovery/gozero v0.0.2 h1:8fJeaCjxL9tpm33uG/RsCQs6HGM/NE6eA3c
github.com/projectdiscovery/gozero v0.0.2/go.mod h1:d8bZvDWW07LWNYWrwjZ4OO1I0cpkfqaysyDfSs9ibK8=
github.com/projectdiscovery/hmap v0.0.44 h1:bFUmPltPPZH+oGDnbYLEXNujZe+0j81c1E+gF1MMLXQ=
github.com/projectdiscovery/hmap v0.0.44/go.mod h1:glLxv5kLMj/Rgn6iZ9JiSzs72t8FQ4hA34zVoFwFhuM=
-github.com/projectdiscovery/httpx v1.6.1 h1:aqKzoSydzKAJBV3Sxbqt44nTah4fNS9GkSD4Dy6Fzlc=
-github.com/projectdiscovery/httpx v1.6.1/go.mod h1:RvibGQbm2Bh2DtuBDVXHLfauFDLtp7opKHyk65sz0ss=
+github.com/projectdiscovery/httpx v1.6.3 h1:TSu3zEKOHGFfMsGeB0lZEFRl6Tq4nPuIgKDXi9NnYpg=
+github.com/projectdiscovery/httpx v1.6.3/go.mod h1:9FmsmkT71aTa4WnOyBIPzB8I9c0QAJ9PSVUqs2NkHRQ=
github.com/projectdiscovery/interactsh v1.1.9 h1:b77SaSGrO+DtivmWwqGGY2dmNlQC3Zgmwlaj9L4Oqvc=
github.com/projectdiscovery/interactsh v1.1.9/go.mod h1:0FRQXCildcTLq7Tsa4BVZAsFCXhpWs4C4quKWigXb5I=
github.com/projectdiscovery/ldapserver v1.0.2-0.20240219154113-dcc758ebc0cb h1:MGtI4oE12ruWv11ZlPXXd7hl/uAaQZrFvrIDYDeVMd8=
@@ -882,12 +882,12 @@ github.com/projectdiscovery/tlsx v1.1.6 h1:iw2zwKbd2+kRQ8J1G4dLmS0CLyemd/tKz1Uzc
github.com/projectdiscovery/tlsx v1.1.6/go.mod h1:s7SRRFdrwIZBK/RXXZi4CR/CubqFSvp8h5Bk1srEZIo=
github.com/projectdiscovery/uncover v1.0.8 h1:nE8bYJuwhqk0BEMRQRhNbjPGMy40A9gkiMk5xq6U4u0=
github.com/projectdiscovery/uncover v1.0.8/go.mod h1:1SwsNqjwMkJEzJQ7lQr5AHIdYd+BQlyqhO5IwIYmIAM=
-github.com/projectdiscovery/useragent v0.0.52 h1:9SUPH0Epo3DJfB6PCDgETfMaD6nZ08sFvfgXTmPUAsU=
-github.com/projectdiscovery/useragent v0.0.52/go.mod h1:PUXHgShvaD8p3bihy1mY8tuBDhdk3M0yy4Z10Ajg2yQ=
+github.com/projectdiscovery/useragent v0.0.53 h1:BKe6z9vb+ajNqmItupjuMO0B6ulG1xC26RsVgEE5p/k=
+github.com/projectdiscovery/useragent v0.0.53/go.mod h1:rToNGuRWpwXACjdL34DuuI0LLHPrCwfWwYHeWMvtPkw=
github.com/projectdiscovery/utils v0.1.0 h1:r7Z/s2CBktJ0bnSN410lzOhD8S/0IxmzmFxkQudYKps=
github.com/projectdiscovery/utils v0.1.0/go.mod h1:RaBdJLTKF5FKZ/RtMeccqFBtpsSjaggVw6/oPTpDD40=
-github.com/projectdiscovery/wappalyzergo v0.1.0 h1:ZagOIKemBsNfCDRQeWavWEXtEjP8UiziuoRoPTwDAJ0=
-github.com/projectdiscovery/wappalyzergo v0.1.0/go.mod h1:wBYGKmA5BQp/NWsAy1q/jSH8N1LHWQ/LV26DuR+KzPM=
+github.com/projectdiscovery/wappalyzergo v0.1.1 h1:HDxuqawatylDiOlfJf4IsabS0wA/Iyvqm7Dn18TVGjU=
+github.com/projectdiscovery/wappalyzergo v0.1.1/go.mod h1:wBYGKmA5BQp/NWsAy1q/jSH8N1LHWQ/LV26DuR+KzPM=
github.com/projectdiscovery/yamldoc-go v1.0.4 h1:eZoESapnMw6WAHiVgRwNqvbJEfNHEH148uthhFbG5jE=
github.com/projectdiscovery/yamldoc-go v1.0.4/go.mod h1:8PIPRcUD55UbtQdcfFR1hpIGRWG0P7alClXNGt1TBik=
github.com/prometheus/client_golang v0.9.1/go.mod h1:7SWBe2y4D6OKWSNQJUaRYU/AaXPKyh/dDVn+NZz0KFw=
From 464aa8de826f8c07b3908e1478259b334caba6f6 Mon Sep 17 00:00:00 2001
From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com>
Date: Mon, 3 Jun 2024 06:20:39 +0000
Subject: [PATCH 034/283] chore(deps): bump github.com/projectdiscovery/hmap
from 0.0.44 to 0.0.45
Bumps [github.com/projectdiscovery/hmap](https://github.com/projectdiscovery/hmap) from 0.0.44 to 0.0.45.
- [Release notes](https://github.com/projectdiscovery/hmap/releases)
- [Commits](https://github.com/projectdiscovery/hmap/compare/v0.0.44...v0.0.45)
---
updated-dependencies:
- dependency-name: github.com/projectdiscovery/hmap
dependency-type: direct:production
update-type: version-update:semver-patch
...
Signed-off-by: dependabot[bot]
---
go.mod | 2 +-
go.sum | 4 ++--
2 files changed, 3 insertions(+), 3 deletions(-)
diff --git a/go.mod b/go.mod
index 729913d2cc..61be9c7fdb 100644
--- a/go.mod
+++ b/go.mod
@@ -21,7 +21,7 @@ require (
github.com/pkg/errors v0.9.1
github.com/projectdiscovery/clistats v0.0.20
github.com/projectdiscovery/fastdialer v0.1.0
- github.com/projectdiscovery/hmap v0.0.44
+ github.com/projectdiscovery/hmap v0.0.45
github.com/projectdiscovery/interactsh v1.1.9
github.com/projectdiscovery/rawhttp v0.1.51
github.com/projectdiscovery/retryabledns v1.0.60
diff --git a/go.sum b/go.sum
index e2fbbeda7e..45afe00606 100644
--- a/go.sum
+++ b/go.sum
@@ -848,8 +848,8 @@ github.com/projectdiscovery/gostruct v0.0.2 h1:s8gP8ApugGM4go1pA+sVlPDXaWqNP5BBD
github.com/projectdiscovery/gostruct v0.0.2/go.mod h1:H86peL4HKwMXcQQtEa6lmC8FuD9XFt6gkNR0B/Mu5PE=
github.com/projectdiscovery/gozero v0.0.2 h1:8fJeaCjxL9tpm33uG/RsCQs6HGM/NE6eA3cjkilRQ+E=
github.com/projectdiscovery/gozero v0.0.2/go.mod h1:d8bZvDWW07LWNYWrwjZ4OO1I0cpkfqaysyDfSs9ibK8=
-github.com/projectdiscovery/hmap v0.0.44 h1:bFUmPltPPZH+oGDnbYLEXNujZe+0j81c1E+gF1MMLXQ=
-github.com/projectdiscovery/hmap v0.0.44/go.mod h1:glLxv5kLMj/Rgn6iZ9JiSzs72t8FQ4hA34zVoFwFhuM=
+github.com/projectdiscovery/hmap v0.0.45 h1:D3PinywmpDtIX2lc9bnqaRJoFDPCCxUf4RvHA3Gdr6U=
+github.com/projectdiscovery/hmap v0.0.45/go.mod h1:vJ7HYzhDnOjsEc9d43Q2pIWRJ5QvFXdObTTQvaUZaPk=
github.com/projectdiscovery/httpx v1.6.3 h1:TSu3zEKOHGFfMsGeB0lZEFRl6Tq4nPuIgKDXi9NnYpg=
github.com/projectdiscovery/httpx v1.6.3/go.mod h1:9FmsmkT71aTa4WnOyBIPzB8I9c0QAJ9PSVUqs2NkHRQ=
github.com/projectdiscovery/interactsh v1.1.9 h1:b77SaSGrO+DtivmWwqGGY2dmNlQC3Zgmwlaj9L4Oqvc=
From bcd10b41e56a587ff034b537af6f53131025ae3a Mon Sep 17 00:00:00 2001
From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com>
Date: Mon, 3 Jun 2024 06:22:45 +0000
Subject: [PATCH 035/283] chore(deps): bump
github.com/projectdiscovery/useragent
Bumps [github.com/projectdiscovery/useragent](https://github.com/projectdiscovery/useragent) from 0.0.52 to 0.0.54.
- [Release notes](https://github.com/projectdiscovery/useragent/releases)
- [Commits](https://github.com/projectdiscovery/useragent/compare/v0.0.52...v0.0.54)
---
updated-dependencies:
- dependency-name: github.com/projectdiscovery/useragent
dependency-type: direct:production
update-type: version-update:semver-patch
...
Signed-off-by: dependabot[bot]
---
go.mod | 2 +-
go.sum | 4 ++--
2 files changed, 3 insertions(+), 3 deletions(-)
diff --git a/go.mod b/go.mod
index 729913d2cc..35ea634bce 100644
--- a/go.mod
+++ b/go.mod
@@ -93,7 +93,7 @@ require (
github.com/projectdiscovery/sarif v0.0.1
github.com/projectdiscovery/tlsx v1.1.6
github.com/projectdiscovery/uncover v1.0.8
- github.com/projectdiscovery/useragent v0.0.53
+ github.com/projectdiscovery/useragent v0.0.54
github.com/projectdiscovery/utils v0.1.0
github.com/projectdiscovery/wappalyzergo v0.1.1
github.com/redis/go-redis/v9 v9.1.0
diff --git a/go.sum b/go.sum
index e2fbbeda7e..868d04fcc2 100644
--- a/go.sum
+++ b/go.sum
@@ -882,8 +882,8 @@ github.com/projectdiscovery/tlsx v1.1.6 h1:iw2zwKbd2+kRQ8J1G4dLmS0CLyemd/tKz1Uzc
github.com/projectdiscovery/tlsx v1.1.6/go.mod h1:s7SRRFdrwIZBK/RXXZi4CR/CubqFSvp8h5Bk1srEZIo=
github.com/projectdiscovery/uncover v1.0.8 h1:nE8bYJuwhqk0BEMRQRhNbjPGMy40A9gkiMk5xq6U4u0=
github.com/projectdiscovery/uncover v1.0.8/go.mod h1:1SwsNqjwMkJEzJQ7lQr5AHIdYd+BQlyqhO5IwIYmIAM=
-github.com/projectdiscovery/useragent v0.0.53 h1:BKe6z9vb+ajNqmItupjuMO0B6ulG1xC26RsVgEE5p/k=
-github.com/projectdiscovery/useragent v0.0.53/go.mod h1:rToNGuRWpwXACjdL34DuuI0LLHPrCwfWwYHeWMvtPkw=
+github.com/projectdiscovery/useragent v0.0.54 h1:3gM4TnhLSspSM/XkI97YcUazXA3KOkQXEU7TIBly2Fc=
+github.com/projectdiscovery/useragent v0.0.54/go.mod h1:dKFkqUb44qx7MpwQDAbhL1t2IrFhcrgI5MlXUZ9E8dM=
github.com/projectdiscovery/utils v0.1.0 h1:r7Z/s2CBktJ0bnSN410lzOhD8S/0IxmzmFxkQudYKps=
github.com/projectdiscovery/utils v0.1.0/go.mod h1:RaBdJLTKF5FKZ/RtMeccqFBtpsSjaggVw6/oPTpDD40=
github.com/projectdiscovery/wappalyzergo v0.1.1 h1:HDxuqawatylDiOlfJf4IsabS0wA/Iyvqm7Dn18TVGjU=
From 04a630266e440e46023ae9fd2bd78ee9995e56ec Mon Sep 17 00:00:00 2001
From: Ikko Eltociear Ashimine
Date: Sat, 8 Jun 2024 21:06:01 +0900
Subject: [PATCH 036/283] docs: add Japanese README (#5259)
* docs: add Japanese README
* docs: update README.md
---
README.md | 4 +-
README_JP.md | 172 +++++++++++++++++++++++++++++++++++++++++++++++++++
2 files changed, 175 insertions(+), 1 deletion(-)
create mode 100644 README_JP.md
diff --git a/README.md b/README.md
index 661dcd626c..54242fd944 100644
--- a/README.md
+++ b/README.md
@@ -31,7 +31,9 @@
中文 •
Korean •
Indonesia •
- Spanish
+ Spanish •
+ 日本語
+
---
diff --git a/README_JP.md b/README_JP.md
new file mode 100644
index 0000000000..f1988e3569
--- /dev/null
+++ b/README_JP.md
@@ -0,0 +1,172 @@
+
+
+
+
+
+シンプルなYAMLベースのDSLに基づいた高速でカスタマイズ可能な脆弱性スキャナー
+
+
+
+
+
+
+
+
+
+
+
+
+
+ 動作原理 •
+ インストール •
+ ドキュメント •
+ クレジット •
+ FAQ •
+ Discordに参加
+
+
+
+ 英語 •
+ 中国語 •
+ 韓国語 •
+ インドネシア語 •
+ スペイン語
+
+
+---
+
+Nucleiは、テンプレートに基づいてターゲット間でリクエストを送信するために使用され、偽陽性がゼロであり、多数のホストで高速なスキャンを提供します。Nucleiは、TCP、DNS、HTTP、SSL、ファイル、Whois、Websocket、Headless、Codeなど、さまざまなプロトコルのスキャンを提供します。強力で柔軟なテンプレートを使用して、Nucleiはすべての種類のセキュリティチェックをモデル化するために使用できます。
+
+**300人以上の** セキュリティ研究者およびエンジニアが提供するさまざまなタイプの脆弱性テンプレートを収容する[専用リポジトリ](https://github.com/projectdiscovery/nuclei-templates)を持っています。
+
+## 動作原理
+
+
+
+
+
+| :exclamation: **免責事項** |
+|---------------------------------|
+| **このプロジェクトは積極的に開発されています**。リリースによって重大な変更が発生することがあります。更新する前にリリースの変更ログを確認してください。 |
+| このプロジェクトは主にスタンドアロンのCLIツールとして使用されることを目的として構築されました。**Nucleiをサービスとして実行すると、セキュリティリスクが生じる可能性があります。**注意して使用し、追加のセキュリティ対策を講じることをお勧めします。 |
+
+# Nucleiのインストール
+
+Nucleiを正常にインストールするには、**go1.21**が必要です。以下のコマンドを実行して最新バージョンをインストールしてください -
+
+```sh
+go install -v github.com/projectdiscovery/nuclei/v3/cmd/nuclei@latest
+```
+
+
+ Brew
+
+ ```sh
+ brew install nuclei
+ ```
+
+
+
+ Docker
+
+ ```sh
+ docker pull projectdiscovery/nuclei:latest
+ ```
+
+
+
+**より多くのインストール方法は[こちら](https://docs.projectdiscovery.io/tools/nuclei/install)で見つけることができます。**
+
+
+
+
+
+### Nucleiテンプレート
+
+Nucleiは、バージョン[v2.5.2](https://github.com/projectdiscovery/nuclei/releases/tag/v2.5.2)以降、デフォルトでテンプレートの自動ダウンロード/更新をサポートしています。[**Nuclei-Templates**](https://github.com/projectdiscovery/nuclei-templates)プロジェクトは、常に更新されるコミュニティ提供の即時使用可能なテンプレートのリストを提供します。
+
+`update-templates`フラグを使用して、いつでもNucleiテンプレートを更新することができます。Nucleiの[テンプレートガイド](https://docs.projectdiscovery.io/templates/)に従って、個々のワークフローとニーズに合わせた独自のチェックを作成することができます。
+
+YAML DSLの構文リファレンスは[こちら](SYNTAX-REFERENCE.md)で確認できます。
+
+ |
+
+
+
+### 使用方法
+
+```sh
+nuclei -h
+```
+
+これにより、ツールのヘルプが表示されます。ここには、サポートされているすべてのスイッチがあります。
+
+```console
+Nucleiは、広範な設定可能性、大規模な拡張性、および使いやすさに焦点を当てた、
+高速でテンプレートベースの脆弱性スキャナーです。
+
+使用法:
+ ./nuclei [flags]
+
+フラグ:
+ターゲット:
+ -u, -target string[] スキャンする対象のURL/ホスト
+ -l, -list string スキャンする対象のURL/ホストのリストが含まれているファイルへのパス(1行に1つ)
+ -resume string resume.cfgを使用してスキャンを再開(クラスタリングは無効になります)
+ -sa, -scan-all-ips DNSレコードに関連付けられているすべてのIPをスキャン
+ -iv, -ip-version string[] ホスト名のスキャンするIPバージョン(4,6)-(デフォルトは4)
+
+テンプレート:
+ -nt, -new-templates 最新のnuclei-templatesリリースに追加された新しいテンプレートのみを実行
+ -ntv, -new-templates-version string[] 特定のバージョンに追加された新しいテンプレートを実行
+ -as, -automatic-scan wappalyzer技術検出をタグマッピングに使用した自動Webスキャン
+ -t, -templates string[] 実行するテンプレートまたはテンプレートディレクトリのリスト(カンマ区切り、ファイル)
+ -turl, -template-url string[] 実行するテンプレートのURLまたはテンプレートURLのリスト(カンマ区切り、ファイル)
+ -w, -workflows string[] 実行するワークフローまたはワークフローディレクトリのリスト(カンマ区切り、ファイル)
+ -wurl, -workflow-url string[] 実行するワークフローのURLまたはワークフローURLのリスト(カンマ区切り、ファイル)
+ -validate Nucleiに渡されたテンプレートを検証
+ -nss, -no-strict-syntax テンプレートで厳密な構文チェックを無効にする
+ -td, -template-display テンプレートの内容を表示
+ -tl 利用可能なすべてのテンプレートをリスト
+ -sign NUCLEI_SIGNATURE_PRIVATE_KEY環境変数で定義された秘密鍵でテンプレートに署名
+ -code コードプロトコルベースのテンプレートのロードを有効にする
+
+フィルタリング:
+ -a, -author string[] 作者に基づいて実行するテンプレート(カンマ区切り、ファイル)
+ -tags string[] タグに基づいて実行するテンプレート(カンマ区切り、ファイル)
+ -etags, -exclude-tags string[] タグに基づいて除外するテンプレート(カンマ区切り、ファイル)
+ -itags, -include-tags string[] デフォルトまたは設定によって除外されている場合でも実行する必要があるタグ
+ -id, -template-id string[] テンプレートIDに基づいて実行するテンプレート(カンマ区切り、ファイル)
+ -eid, -exclude-id string[] テンプレートIDに基づいて除外するテンプレート(カンマ区切り、ファイル)
+ -it, -include-templates string[] デフォルトまたは設定によって除外されている場合でも実行する必要があるテンプレート
+ -et, -exclude-templates string[] 除外するテンプレートまたはテンプレートディレクトリへのパス(カンマ区切り、ファイル)
+ -em, -exclude-matchers string[] 結果で除外するテンプレートマッチャー
+ -s, -severity value[] 重大度に基づいて実行するテンプレート。可能な値:info, low, medium, high, critical, unknown
+ -es, -exclude-severity value[] 重大度に基づいて除外するテンプレート。可能な値:info, low, medium, high, critical, unknown
+ -pt, -type value[] プロトコルタイプに基づいて実行するテンプレート。可能な値:dns, file, http, headless, tcp, workflow, ssl, websocket, whois, code, javascript
+ -ept, -exclude-type value[] プロトコルタイプに基づいて除外するテンプレート。可能な値:dns, file, http, headless, tcp, workflow, ssl, websocket, whois, code, javascript
+ -tc, -template-condition string[] 式条件に基づいて実行するテンプレート
+
+出力:
+ -o, -output string 発見された問題/脆弱性を書き込む出力ファイル
+ -sresp, -store-resp Nucleiを通じて渡されたすべてのリクエスト/レスポンスを出力ディレクトリに保存
+ -srd, -store-resp-dir string Nucleiを通じて渡されたすべてのリクエスト/レスポンスをカスタムディレクトリに保存(デフォルトは「output」)
+ -silent 結果のみを表示
+ -nc, -no-color 出力内容の着色を無効にする(ANSIエスケープコード)
+ -j, -jsonl JSONL(ines)形式で出力を書き込む
+ -irr, -include-rr -omit-raw JSON、JSONL、Markdown出力にリクエスト/レスポンスペアを含める(発見のみ)[非推奨 -omit-raw使用](デフォルトはtrue)
+ -or, -omit-raw JSON、JSONL、Markdown出力でリクエスト/レスポンスペアを省略する(発見のみ)
+ -ot, -omit-template JSON、JSONL出力でエンコードされたテンプレートを省略
+ -nm, -no-meta CLI出力で結果のメタデータの印刷を無効にする
+ -ts, -timestamp CLI出力にタイムスタンプを印刷することを有効にする
+ -rdb, -report-db string Nucleiレポートデータベース(レポートデータを永続化するために常にこれを使用)
+ -ms, -matcher-status マッチ失敗のステータスを表示
+ -me, -markdown-export string Markdown形式で結果をエクスポートするディレクトリ
+ -se, -sarif-export string SARIF形式で結果をエクスポートするファイル
+ -je, -json-export string JSON形式で結果をエクスポートするファイル
+ -jle, -jsonl-export string JSONL(ine)形式で結果をエクスポートするファイル
+
+設定:
+ -config string Nucleiの設定ファイルへのパス
+ -fr, -follow-redirects HTTPテンプレートのリダイレクトをフォローすることを有効にする
+ -fhr, -follow-host-redirects
\ No newline at end of file
From fd2311d034b4da358f8b6759da48bdc4ba509106 Mon Sep 17 00:00:00 2001
From: mzack
Date: Mon, 10 Jun 2024 03:23:00 +0200
Subject: [PATCH 037/283] fixing slow elab
---
pkg/protocols/common/hosterrorscache/hosterrorscache.go | 6 ++++++
1 file changed, 6 insertions(+)
diff --git a/pkg/protocols/common/hosterrorscache/hosterrorscache.go b/pkg/protocols/common/hosterrorscache/hosterrorscache.go
index e40f13dde3..badff95ddd 100644
--- a/pkg/protocols/common/hosterrorscache/hosterrorscache.go
+++ b/pkg/protocols/common/hosterrorscache/hosterrorscache.go
@@ -185,6 +185,12 @@ func (c *Cache) checkError(err error) bool {
if err == nil {
return false
}
+
+ // we do not consider timeouts as temporary
+ if strings.Contains(errkit.FromError(err).Cause().Error(), "i/o timeout") {
+ return true
+ }
+
kind := errkit.GetErrorKind(err, nucleierr.ErrTemplateLogic)
switch kind {
case nucleierr.ErrTemplateLogic:
From f0b34f94cd0535c5270f2c594612c2df384247d3 Mon Sep 17 00:00:00 2001
From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com>
Date: Mon, 10 Jun 2024 05:56:53 +0000
Subject: [PATCH 038/283] chore(deps): bump
github.com/projectdiscovery/retryabledns
Bumps [github.com/projectdiscovery/retryabledns](https://github.com/projectdiscovery/retryabledns) from 1.0.60 to 1.0.63.
- [Release notes](https://github.com/projectdiscovery/retryabledns/releases)
- [Commits](https://github.com/projectdiscovery/retryabledns/compare/v1.0.60...v1.0.63)
---
updated-dependencies:
- dependency-name: github.com/projectdiscovery/retryabledns
dependency-type: direct:production
update-type: version-update:semver-patch
...
Signed-off-by: dependabot[bot]
---
go.mod | 4 ++--
go.sum | 8 ++++----
2 files changed, 6 insertions(+), 6 deletions(-)
diff --git a/go.mod b/go.mod
index 07c36e7d95..4cafa03ed4 100644
--- a/go.mod
+++ b/go.mod
@@ -24,7 +24,7 @@ require (
github.com/projectdiscovery/hmap v0.0.45
github.com/projectdiscovery/interactsh v1.1.9
github.com/projectdiscovery/rawhttp v0.1.51
- github.com/projectdiscovery/retryabledns v1.0.60
+ github.com/projectdiscovery/retryabledns v1.0.63
github.com/projectdiscovery/retryablehttp-go v1.0.62
github.com/projectdiscovery/yamldoc-go v1.0.4
github.com/remeh/sizedwaitgroup v1.0.0
@@ -94,7 +94,7 @@ require (
github.com/projectdiscovery/tlsx v1.1.6
github.com/projectdiscovery/uncover v1.0.8
github.com/projectdiscovery/useragent v0.0.54
- github.com/projectdiscovery/utils v0.1.0
+ github.com/projectdiscovery/utils v0.1.1
github.com/projectdiscovery/wappalyzergo v0.1.1
github.com/redis/go-redis/v9 v9.1.0
github.com/seh-msft/burpxml v1.0.1
diff --git a/go.sum b/go.sum
index ccdec0fcab..a6919fec89 100644
--- a/go.sum
+++ b/go.sum
@@ -870,8 +870,8 @@ github.com/projectdiscovery/rawhttp v0.1.51 h1:T4sskU2fsX9zNR7nyae0tD/0fliIs73Kw
github.com/projectdiscovery/rawhttp v0.1.51/go.mod h1:ejBaw3dCfJSfbtFv0ib6YfFSjQzt7ZdRVpQWJASxIUs=
github.com/projectdiscovery/rdap v0.9.1-0.20221108103045-9865884d1917 h1:m03X4gBVSorSzvmm0bFa7gDV4QNSOWPL/fgZ4kTXBxk=
github.com/projectdiscovery/rdap v0.9.1-0.20221108103045-9865884d1917/go.mod h1:JxXtZC9e195awe7EynrcnBJmFoad/BNDzW9mzFkK8Sg=
-github.com/projectdiscovery/retryabledns v1.0.60 h1:f3TPeLnaJKrl9CbfsTfFTFnJY+UnyArH3mxxUUyR5ZQ=
-github.com/projectdiscovery/retryabledns v1.0.60/go.mod h1:T4Su40Wa9lVtRNMfMDFJi00g2T3FbTfwnKKkYON0WgU=
+github.com/projectdiscovery/retryabledns v1.0.63 h1:Ijt47ybwf+iIwul606NlPKPN+ouhOYPeA9sLHVgqLG4=
+github.com/projectdiscovery/retryabledns v1.0.63/go.mod h1:lTs48OYJnMFuuBzT+3z3PrZ58K0OUBgP7Y4o3ttBwb0=
github.com/projectdiscovery/retryablehttp-go v1.0.62 h1:ZY09NYRatoELHaPUcdRdWzHIJExjEqceHAfe7u/e9ZY=
github.com/projectdiscovery/retryablehttp-go v1.0.62/go.mod h1:yhRy9Q3z0CuNUHIIPrveaubjVEwF4arKWT/5SQurQEE=
github.com/projectdiscovery/sarif v0.0.1 h1:C2Tyj0SGOKbCLgHrx83vaE6YkzXEVrMXYRGLkKCr/us=
@@ -884,8 +884,8 @@ github.com/projectdiscovery/uncover v1.0.8 h1:nE8bYJuwhqk0BEMRQRhNbjPGMy40A9gkiM
github.com/projectdiscovery/uncover v1.0.8/go.mod h1:1SwsNqjwMkJEzJQ7lQr5AHIdYd+BQlyqhO5IwIYmIAM=
github.com/projectdiscovery/useragent v0.0.54 h1:3gM4TnhLSspSM/XkI97YcUazXA3KOkQXEU7TIBly2Fc=
github.com/projectdiscovery/useragent v0.0.54/go.mod h1:dKFkqUb44qx7MpwQDAbhL1t2IrFhcrgI5MlXUZ9E8dM=
-github.com/projectdiscovery/utils v0.1.0 h1:r7Z/s2CBktJ0bnSN410lzOhD8S/0IxmzmFxkQudYKps=
-github.com/projectdiscovery/utils v0.1.0/go.mod h1:RaBdJLTKF5FKZ/RtMeccqFBtpsSjaggVw6/oPTpDD40=
+github.com/projectdiscovery/utils v0.1.1 h1:iQ/DyrClxbIbKMUCXfXLA1lFkqprrUM9Ti/nMU5dQj4=
+github.com/projectdiscovery/utils v0.1.1/go.mod h1:EPuSvVIvp61nXJD5EO65vaCv82OuhO+wfZpWAWA0q3o=
github.com/projectdiscovery/wappalyzergo v0.1.1 h1:HDxuqawatylDiOlfJf4IsabS0wA/Iyvqm7Dn18TVGjU=
github.com/projectdiscovery/wappalyzergo v0.1.1/go.mod h1:wBYGKmA5BQp/NWsAy1q/jSH8N1LHWQ/LV26DuR+KzPM=
github.com/projectdiscovery/yamldoc-go v1.0.4 h1:eZoESapnMw6WAHiVgRwNqvbJEfNHEH148uthhFbG5jE=
From 56e9b7fcbc5b4fff9ced660f6e99dd5b08aee013 Mon Sep 17 00:00:00 2001
From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com>
Date: Mon, 10 Jun 2024 06:24:45 +0000
Subject: [PATCH 039/283] chore(deps): bump
github.com/projectdiscovery/wappalyzergo
Bumps [github.com/projectdiscovery/wappalyzergo](https://github.com/projectdiscovery/wappalyzergo) from 0.1.1 to 0.1.4.
- [Release notes](https://github.com/projectdiscovery/wappalyzergo/releases)
- [Commits](https://github.com/projectdiscovery/wappalyzergo/compare/v0.1.1...v0.1.4)
---
updated-dependencies:
- dependency-name: github.com/projectdiscovery/wappalyzergo
dependency-type: direct:production
update-type: version-update:semver-patch
...
Signed-off-by: dependabot[bot]
---
go.mod | 2 +-
go.sum | 4 ++--
2 files changed, 3 insertions(+), 3 deletions(-)
diff --git a/go.mod b/go.mod
index 4cafa03ed4..8f44d7ca01 100644
--- a/go.mod
+++ b/go.mod
@@ -95,7 +95,7 @@ require (
github.com/projectdiscovery/uncover v1.0.8
github.com/projectdiscovery/useragent v0.0.54
github.com/projectdiscovery/utils v0.1.1
- github.com/projectdiscovery/wappalyzergo v0.1.1
+ github.com/projectdiscovery/wappalyzergo v0.1.4
github.com/redis/go-redis/v9 v9.1.0
github.com/seh-msft/burpxml v1.0.1
github.com/stretchr/testify v1.9.0
diff --git a/go.sum b/go.sum
index a6919fec89..81f7d0dcc1 100644
--- a/go.sum
+++ b/go.sum
@@ -886,8 +886,8 @@ github.com/projectdiscovery/useragent v0.0.54 h1:3gM4TnhLSspSM/XkI97YcUazXA3KOkQ
github.com/projectdiscovery/useragent v0.0.54/go.mod h1:dKFkqUb44qx7MpwQDAbhL1t2IrFhcrgI5MlXUZ9E8dM=
github.com/projectdiscovery/utils v0.1.1 h1:iQ/DyrClxbIbKMUCXfXLA1lFkqprrUM9Ti/nMU5dQj4=
github.com/projectdiscovery/utils v0.1.1/go.mod h1:EPuSvVIvp61nXJD5EO65vaCv82OuhO+wfZpWAWA0q3o=
-github.com/projectdiscovery/wappalyzergo v0.1.1 h1:HDxuqawatylDiOlfJf4IsabS0wA/Iyvqm7Dn18TVGjU=
-github.com/projectdiscovery/wappalyzergo v0.1.1/go.mod h1:wBYGKmA5BQp/NWsAy1q/jSH8N1LHWQ/LV26DuR+KzPM=
+github.com/projectdiscovery/wappalyzergo v0.1.4 h1:Q0j5t/c74DlPKRvGYLZFv8npTA7MBW9uwdlsMBn/Yh4=
+github.com/projectdiscovery/wappalyzergo v0.1.4/go.mod h1:wBYGKmA5BQp/NWsAy1q/jSH8N1LHWQ/LV26DuR+KzPM=
github.com/projectdiscovery/yamldoc-go v1.0.4 h1:eZoESapnMw6WAHiVgRwNqvbJEfNHEH148uthhFbG5jE=
github.com/projectdiscovery/yamldoc-go v1.0.4/go.mod h1:8PIPRcUD55UbtQdcfFR1hpIGRWG0P7alClXNGt1TBik=
github.com/prometheus/client_golang v0.9.1/go.mod h1:7SWBe2y4D6OKWSNQJUaRYU/AaXPKyh/dDVn+NZz0KFw=
From 86dabe1a989a6312e426efb5b49a58a6da3a9719 Mon Sep 17 00:00:00 2001
From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com>
Date: Mon, 10 Jun 2024 06:24:46 +0000
Subject: [PATCH 040/283] chore(deps): bump github.com/projectdiscovery/dsl
from 0.1.0 to 0.1.2
Bumps [github.com/projectdiscovery/dsl](https://github.com/projectdiscovery/dsl) from 0.1.0 to 0.1.2.
- [Release notes](https://github.com/projectdiscovery/dsl/releases)
- [Commits](https://github.com/projectdiscovery/dsl/compare/v0.1.0...v0.1.2)
---
updated-dependencies:
- dependency-name: github.com/projectdiscovery/dsl
dependency-type: direct:production
update-type: version-update:semver-patch
...
Signed-off-by: dependabot[bot]
---
go.mod | 2 +-
go.sum | 4 ++--
2 files changed, 3 insertions(+), 3 deletions(-)
diff --git a/go.mod b/go.mod
index 4cafa03ed4..532a3a3b84 100644
--- a/go.mod
+++ b/go.mod
@@ -78,7 +78,7 @@ require (
github.com/mholt/archiver v3.1.1+incompatible
github.com/ory/dockertest/v3 v3.10.0
github.com/praetorian-inc/fingerprintx v1.1.9
- github.com/projectdiscovery/dsl v0.1.0
+ github.com/projectdiscovery/dsl v0.1.2
github.com/projectdiscovery/fasttemplate v0.0.2
github.com/projectdiscovery/go-smb2 v0.0.0-20240129202741-052cc450c6cb
github.com/projectdiscovery/goflags v0.1.54
diff --git a/go.sum b/go.sum
index a6919fec89..c935991558 100644
--- a/go.sum
+++ b/go.sum
@@ -830,8 +830,8 @@ github.com/projectdiscovery/cdncheck v1.0.9 h1:BS15gzj9gb5AVSKqTDzPamfSgStu7nJQO
github.com/projectdiscovery/cdncheck v1.0.9/go.mod h1:18SSl1w7rMj53CGeRIZTbDoa286a6xZIxGbaiEo4Fxs=
github.com/projectdiscovery/clistats v0.0.20 h1:5jO5SLiRJ7f0nDV0ndBNmBeesbROouPooH+DGMgoWq4=
github.com/projectdiscovery/clistats v0.0.20/go.mod h1:GJ2av0KnOvK0AISQnP8hyDclYIji1LVkx2l0pwnzAu4=
-github.com/projectdiscovery/dsl v0.1.0 h1:10xoL0V/btU646xH6ioGXKwrGt6grT+BIa6id1/4ifE=
-github.com/projectdiscovery/dsl v0.1.0/go.mod h1:JgOjLk54Tl9HG98N2SM5OaKKJ/STpI7yDiKW5rhV+yg=
+github.com/projectdiscovery/dsl v0.1.2 h1:RdWCEsUqyWsLIQ1k16NX5FiIVbPFvzx9BOlvzK7eeGw=
+github.com/projectdiscovery/dsl v0.1.2/go.mod h1:3zwogxGPnPkVUk5OfZQ9Y51sAWdazfQoVZFSPqeEUYA=
github.com/projectdiscovery/fastdialer v0.1.0 h1:NiId7eXvYZjiBo83AnIZ6K8ZjzLJU8wGXdRLzuKcRrg=
github.com/projectdiscovery/fastdialer v0.1.0/go.mod h1:jKOWzyHx+Q2sMiYst5HP6tWLQLDFPVgWBR72szn2h8w=
github.com/projectdiscovery/fasttemplate v0.0.2 h1:h2cISk5xDhlJEinlBQS6RRx0vOlOirB2y3Yu4PJzpiA=
From aced7ae42a9b34f85be98f50daef59067b0998b5 Mon Sep 17 00:00:00 2001
From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com>
Date: Mon, 10 Jun 2024 06:53:41 +0000
Subject: [PATCH 041/283] chore(deps): bump
github.com/projectdiscovery/interactsh
Bumps [github.com/projectdiscovery/interactsh](https://github.com/projectdiscovery/interactsh) from 1.1.9 to 1.2.0.
- [Release notes](https://github.com/projectdiscovery/interactsh/releases)
- [Changelog](https://github.com/projectdiscovery/interactsh/blob/main/.goreleaser.yml)
- [Commits](https://github.com/projectdiscovery/interactsh/compare/v1.1.9...v1.2.0)
---
updated-dependencies:
- dependency-name: github.com/projectdiscovery/interactsh
dependency-type: direct:production
...
Signed-off-by: dependabot[bot]
---
go.mod | 6 +++---
go.sum | 12 ++++++------
2 files changed, 9 insertions(+), 9 deletions(-)
diff --git a/go.mod b/go.mod
index efe52be206..6b106a7aba 100644
--- a/go.mod
+++ b/go.mod
@@ -20,12 +20,12 @@ require (
github.com/olekukonko/tablewriter v0.0.5
github.com/pkg/errors v0.9.1
github.com/projectdiscovery/clistats v0.0.20
- github.com/projectdiscovery/fastdialer v0.1.0
+ github.com/projectdiscovery/fastdialer v0.1.1
github.com/projectdiscovery/hmap v0.0.45
- github.com/projectdiscovery/interactsh v1.1.9
+ github.com/projectdiscovery/interactsh v1.2.0
github.com/projectdiscovery/rawhttp v0.1.51
github.com/projectdiscovery/retryabledns v1.0.63
- github.com/projectdiscovery/retryablehttp-go v1.0.62
+ github.com/projectdiscovery/retryablehttp-go v1.0.63
github.com/projectdiscovery/yamldoc-go v1.0.4
github.com/remeh/sizedwaitgroup v1.0.0
github.com/rs/xid v1.5.0
diff --git a/go.sum b/go.sum
index 46f1f8ac74..8d4bbeddbc 100644
--- a/go.sum
+++ b/go.sum
@@ -832,8 +832,8 @@ github.com/projectdiscovery/clistats v0.0.20 h1:5jO5SLiRJ7f0nDV0ndBNmBeesbROouPo
github.com/projectdiscovery/clistats v0.0.20/go.mod h1:GJ2av0KnOvK0AISQnP8hyDclYIji1LVkx2l0pwnzAu4=
github.com/projectdiscovery/dsl v0.1.2 h1:RdWCEsUqyWsLIQ1k16NX5FiIVbPFvzx9BOlvzK7eeGw=
github.com/projectdiscovery/dsl v0.1.2/go.mod h1:3zwogxGPnPkVUk5OfZQ9Y51sAWdazfQoVZFSPqeEUYA=
-github.com/projectdiscovery/fastdialer v0.1.0 h1:NiId7eXvYZjiBo83AnIZ6K8ZjzLJU8wGXdRLzuKcRrg=
-github.com/projectdiscovery/fastdialer v0.1.0/go.mod h1:jKOWzyHx+Q2sMiYst5HP6tWLQLDFPVgWBR72szn2h8w=
+github.com/projectdiscovery/fastdialer v0.1.1 h1:51IovI02Ime7fVCWFxMpg1akCcaRHpVQnjpTQT0+0G0=
+github.com/projectdiscovery/fastdialer v0.1.1/go.mod h1:tIMAT93FmV90EoVfM1lfaTrkNEbqT+G4jLyyoOO5McY=
github.com/projectdiscovery/fasttemplate v0.0.2 h1:h2cISk5xDhlJEinlBQS6RRx0vOlOirB2y3Yu4PJzpiA=
github.com/projectdiscovery/fasttemplate v0.0.2/go.mod h1:XYWWVMxnItd+r0GbjA1GCsUopMw1/XusuQxdyAIHMCw=
github.com/projectdiscovery/freeport v0.0.5 h1:jnd3Oqsl4S8n0KuFkE5Hm8WGDP24ITBvmyw5pFTHS8Q=
@@ -852,8 +852,8 @@ github.com/projectdiscovery/hmap v0.0.45 h1:D3PinywmpDtIX2lc9bnqaRJoFDPCCxUf4RvH
github.com/projectdiscovery/hmap v0.0.45/go.mod h1:vJ7HYzhDnOjsEc9d43Q2pIWRJ5QvFXdObTTQvaUZaPk=
github.com/projectdiscovery/httpx v1.6.3 h1:TSu3zEKOHGFfMsGeB0lZEFRl6Tq4nPuIgKDXi9NnYpg=
github.com/projectdiscovery/httpx v1.6.3/go.mod h1:9FmsmkT71aTa4WnOyBIPzB8I9c0QAJ9PSVUqs2NkHRQ=
-github.com/projectdiscovery/interactsh v1.1.9 h1:b77SaSGrO+DtivmWwqGGY2dmNlQC3Zgmwlaj9L4Oqvc=
-github.com/projectdiscovery/interactsh v1.1.9/go.mod h1:0FRQXCildcTLq7Tsa4BVZAsFCXhpWs4C4quKWigXb5I=
+github.com/projectdiscovery/interactsh v1.2.0 h1:Al6jHiR+Usl9egYJDLJaWNHOcH8Rugk8gWMasc8Cmw8=
+github.com/projectdiscovery/interactsh v1.2.0/go.mod h1:Wxt0fnzxsfrAZQQlpVrf3xMatP4OXZaZbjuDkIQKdYY=
github.com/projectdiscovery/ldapserver v1.0.2-0.20240219154113-dcc758ebc0cb h1:MGtI4oE12ruWv11ZlPXXd7hl/uAaQZrFvrIDYDeVMd8=
github.com/projectdiscovery/ldapserver v1.0.2-0.20240219154113-dcc758ebc0cb/go.mod h1:vmgC0DTFCfoCLp0RAfsfYTZZan0QMVs+cmTbH6blfjk=
github.com/projectdiscovery/machineid v0.0.0-20240226150047-2e2c51e35983 h1:ZScLodGSezQVwsQDtBSMFp72WDq0nNN+KE/5DHKY5QE=
@@ -872,8 +872,8 @@ github.com/projectdiscovery/rdap v0.9.1-0.20221108103045-9865884d1917 h1:m03X4gB
github.com/projectdiscovery/rdap v0.9.1-0.20221108103045-9865884d1917/go.mod h1:JxXtZC9e195awe7EynrcnBJmFoad/BNDzW9mzFkK8Sg=
github.com/projectdiscovery/retryabledns v1.0.63 h1:Ijt47ybwf+iIwul606NlPKPN+ouhOYPeA9sLHVgqLG4=
github.com/projectdiscovery/retryabledns v1.0.63/go.mod h1:lTs48OYJnMFuuBzT+3z3PrZ58K0OUBgP7Y4o3ttBwb0=
-github.com/projectdiscovery/retryablehttp-go v1.0.62 h1:ZY09NYRatoELHaPUcdRdWzHIJExjEqceHAfe7u/e9ZY=
-github.com/projectdiscovery/retryablehttp-go v1.0.62/go.mod h1:yhRy9Q3z0CuNUHIIPrveaubjVEwF4arKWT/5SQurQEE=
+github.com/projectdiscovery/retryablehttp-go v1.0.63 h1:y/MpWrAO7702ybOLo/OuyAx6bZ6NJVEf85mGMAU51l8=
+github.com/projectdiscovery/retryablehttp-go v1.0.63/go.mod h1:ytE1KGi63NFAyLzWqTwZJXwjAg7apgqTnJtaZeZ4qtM=
github.com/projectdiscovery/sarif v0.0.1 h1:C2Tyj0SGOKbCLgHrx83vaE6YkzXEVrMXYRGLkKCr/us=
github.com/projectdiscovery/sarif v0.0.1/go.mod h1:cEYlDu8amcPf6b9dSakcz2nNnJsoz4aR6peERwV+wuQ=
github.com/projectdiscovery/stringsutil v0.0.2 h1:uzmw3IVLJSMW1kEg8eCStG/cGbYYZAja8BH3LqqJXMA=
From df57f0a6e3dac2db5a8d563e7bcc1c7c8f1264bd Mon Sep 17 00:00:00 2001
From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com>
Date: Mon, 10 Jun 2024 07:24:01 +0000
Subject: [PATCH 042/283] chore(deps): bump
github.com/projectdiscovery/fastdialer
Bumps [github.com/projectdiscovery/fastdialer](https://github.com/projectdiscovery/fastdialer) from 0.1.0 to 0.1.3.
- [Release notes](https://github.com/projectdiscovery/fastdialer/releases)
- [Commits](https://github.com/projectdiscovery/fastdialer/compare/v0.1.0...v0.1.3)
---
updated-dependencies:
- dependency-name: github.com/projectdiscovery/fastdialer
dependency-type: direct:production
update-type: version-update:semver-patch
...
Signed-off-by: dependabot[bot]
---
go.mod | 2 +-
go.sum | 4 ++--
2 files changed, 3 insertions(+), 3 deletions(-)
diff --git a/go.mod b/go.mod
index 6b106a7aba..eab2fea4c9 100644
--- a/go.mod
+++ b/go.mod
@@ -20,7 +20,7 @@ require (
github.com/olekukonko/tablewriter v0.0.5
github.com/pkg/errors v0.9.1
github.com/projectdiscovery/clistats v0.0.20
- github.com/projectdiscovery/fastdialer v0.1.1
+ github.com/projectdiscovery/fastdialer v0.1.3
github.com/projectdiscovery/hmap v0.0.45
github.com/projectdiscovery/interactsh v1.2.0
github.com/projectdiscovery/rawhttp v0.1.51
diff --git a/go.sum b/go.sum
index 8d4bbeddbc..a0ea94c394 100644
--- a/go.sum
+++ b/go.sum
@@ -832,8 +832,8 @@ github.com/projectdiscovery/clistats v0.0.20 h1:5jO5SLiRJ7f0nDV0ndBNmBeesbROouPo
github.com/projectdiscovery/clistats v0.0.20/go.mod h1:GJ2av0KnOvK0AISQnP8hyDclYIji1LVkx2l0pwnzAu4=
github.com/projectdiscovery/dsl v0.1.2 h1:RdWCEsUqyWsLIQ1k16NX5FiIVbPFvzx9BOlvzK7eeGw=
github.com/projectdiscovery/dsl v0.1.2/go.mod h1:3zwogxGPnPkVUk5OfZQ9Y51sAWdazfQoVZFSPqeEUYA=
-github.com/projectdiscovery/fastdialer v0.1.1 h1:51IovI02Ime7fVCWFxMpg1akCcaRHpVQnjpTQT0+0G0=
-github.com/projectdiscovery/fastdialer v0.1.1/go.mod h1:tIMAT93FmV90EoVfM1lfaTrkNEbqT+G4jLyyoOO5McY=
+github.com/projectdiscovery/fastdialer v0.1.3 h1:DuXMFFDnhHP2pXWYZXqZHS76vSvkMeRYD4NifI/DlXI=
+github.com/projectdiscovery/fastdialer v0.1.3/go.mod h1:V5fkUburMLylbtBqbei5aAXTt1foDMeX517iM3uPkao=
github.com/projectdiscovery/fasttemplate v0.0.2 h1:h2cISk5xDhlJEinlBQS6RRx0vOlOirB2y3Yu4PJzpiA=
github.com/projectdiscovery/fasttemplate v0.0.2/go.mod h1:XYWWVMxnItd+r0GbjA1GCsUopMw1/XusuQxdyAIHMCw=
github.com/projectdiscovery/freeport v0.0.5 h1:jnd3Oqsl4S8n0KuFkE5Hm8WGDP24ITBvmyw5pFTHS8Q=
From 9f3f7fce0665dd69eefb714b8bc2296b49f2b8bd Mon Sep 17 00:00:00 2001
From: Ice3man
Date: Tue, 11 Jun 2024 04:43:46 +0530
Subject: [PATCH 043/283] Fuzzing additions & enhancements (#5139)
* feat: added fuzzing output enhancements
* changes as requested
* misc
* feat: added dfp flag to display fuzz points + misc additions
* feat: added support for fuzzing nested path segments
* feat: added parts to fuzzing requests
* feat: added tracking for parameter occurence frequency in fuzzing
* added cli flag for fuzz frequency
* fixed broken tests
* fixed path based sqli integration test
* feat: added configurable fuzzing aggression level for payloads
* fixed failing test
---
cmd/nuclei/main.go | 3 +
integration_tests/fuzz/fuzz-path-sqli.yaml | 7 +-
internal/runner/runner.go | 67 ++++----
pkg/fuzz/component/path.go | 50 ++++--
pkg/fuzz/component/path_test.go | 40 ++++-
pkg/fuzz/component/query.go | 1 +
pkg/fuzz/execute.go | 27 +++-
pkg/fuzz/frequency/tracker.go | 152 ++++++++++++++++++
pkg/fuzz/fuzz.go | 19 ++-
pkg/fuzz/fuzz_test.go | 8 +-
pkg/fuzz/parts.go | 28 +++-
pkg/protocols/common/generators/generators.go | 69 +++++++-
.../common/generators/generators_test.go | 48 ++++++
pkg/protocols/common/interactsh/interactsh.go | 12 ++
pkg/protocols/common/interactsh/options.go | 6 +-
pkg/protocols/http/request_fuzz.go | 15 +-
pkg/protocols/protocols.go | 3 +
pkg/testutils/fuzzplayground/server.go | 22 ++-
pkg/types/types.go | 6 +
19 files changed, 507 insertions(+), 76 deletions(-)
create mode 100644 pkg/fuzz/frequency/tracker.go
diff --git a/cmd/nuclei/main.go b/cmd/nuclei/main.go
index e93a4c4441..40984074df 100644
--- a/cmd/nuclei/main.go
+++ b/cmd/nuclei/main.go
@@ -326,6 +326,9 @@ on extensive configurability, massive extensibility and ease of use.`)
flagSet.StringVarP(&options.FuzzingMode, "fuzzing-mode", "fm", "", "overrides fuzzing mode set in template (multiple, single)"),
flagSet.BoolVar(&fuzzFlag, "fuzz", false, "enable loading fuzzing templates (Deprecated: use -dast instead)"),
flagSet.BoolVar(&options.DAST, "dast", false, "enable / run dast (fuzz) nuclei templates"),
+ flagSet.BoolVarP(&options.DisplayFuzzPoints, "display-fuzz-points", "dfp", false, "display fuzz points in the output for debugging"),
+ flagSet.IntVar(&options.FuzzParamFrequency, "fuzz-param-frequency", 10, "frequency of uninteresting parameters for fuzzing before skipping"),
+ flagSet.StringVarP(&options.FuzzAggressionLevel, "fuzz-aggression", "fa", "low", "fuzzing aggression level controls payload count for fuzz (low, medium, high)"),
)
flagSet.CreateGroup("uncover", "Uncover",
diff --git a/integration_tests/fuzz/fuzz-path-sqli.yaml b/integration_tests/fuzz/fuzz-path-sqli.yaml
index e23098d931..531427becb 100644
--- a/integration_tests/fuzz/fuzz-path-sqli.yaml
+++ b/integration_tests/fuzz/fuzz-path-sqli.yaml
@@ -15,21 +15,18 @@ http:
- type: dsl
dsl:
- 'method == "GET"'
- - regex("/(.*?/)([0-9]+)(/.*)?",path)
condition: and
payloads:
pathsqli:
- - "'OR1=1"
- '%20OR%20True'
fuzzing:
- part: path
- type: replace-regex
+ type: postfix
mode: single
- replace-regex: '/(.*?/)([0-9]+)(/.*)?'
fuzz:
- - '/${1}${2}{{pathsqli}}${3}'
+ - '{{pathsqli}}'
matchers:
- type: status
diff --git a/internal/runner/runner.go b/internal/runner/runner.go
index 24010d7537..bc436500af 100644
--- a/internal/runner/runner.go
+++ b/internal/runner/runner.go
@@ -15,6 +15,7 @@ import (
"github.com/projectdiscovery/nuclei/v3/internal/pdcp"
"github.com/projectdiscovery/nuclei/v3/pkg/authprovider"
+ "github.com/projectdiscovery/nuclei/v3/pkg/fuzz/frequency"
"github.com/projectdiscovery/nuclei/v3/pkg/input/provider"
"github.com/projectdiscovery/nuclei/v3/pkg/installer"
"github.com/projectdiscovery/nuclei/v3/pkg/loader/parser"
@@ -74,21 +75,22 @@ var (
// Runner is a client for running the enumeration process.
type Runner struct {
- output output.Writer
- interactsh *interactsh.Client
- options *types.Options
- projectFile *projectfile.ProjectFile
- catalog catalog.Catalog
- progress progress.Progress
- colorizer aurora.Aurora
- issuesClient reporting.Client
- browser *engine.Browser
- rateLimiter *ratelimit.Limiter
- hostErrors hosterrorscache.CacheInterface
- resumeCfg *types.ResumeCfg
- pprofServer *http.Server
- pdcpUploadErrMsg string
- inputProvider provider.InputProvider
+ output output.Writer
+ interactsh *interactsh.Client
+ options *types.Options
+ projectFile *projectfile.ProjectFile
+ catalog catalog.Catalog
+ progress progress.Progress
+ colorizer aurora.Aurora
+ issuesClient reporting.Client
+ browser *engine.Browser
+ rateLimiter *ratelimit.Limiter
+ hostErrors hosterrorscache.CacheInterface
+ resumeCfg *types.ResumeCfg
+ pprofServer *http.Server
+ pdcpUploadErrMsg string
+ inputProvider provider.InputProvider
+ fuzzFrequencyCache *frequency.Tracker
//general purpose temporary directory
tmpDir string
parser parser.Parser
@@ -448,24 +450,28 @@ func (r *Runner) RunEnumeration() error {
r.options.ExcludedTemplates = append(r.options.ExcludedTemplates, ignoreFile.Files...)
}
+ fuzzFreqCache := frequency.New(frequency.DefaultMaxTrackCount, r.options.FuzzParamFrequency)
+ r.fuzzFrequencyCache = fuzzFreqCache
+
// Create the executor options which will be used throughout the execution
// stage by the nuclei engine modules.
executorOpts := protocols.ExecutorOptions{
- Output: r.output,
- Options: r.options,
- Progress: r.progress,
- Catalog: r.catalog,
- IssuesClient: r.issuesClient,
- RateLimiter: r.rateLimiter,
- Interactsh: r.interactsh,
- ProjectFile: r.projectFile,
- Browser: r.browser,
- Colorizer: r.colorizer,
- ResumeCfg: r.resumeCfg,
- ExcludeMatchers: excludematchers.New(r.options.ExcludeMatchers),
- InputHelper: input.NewHelper(),
- TemporaryDirectory: r.tmpDir,
- Parser: r.parser,
+ Output: r.output,
+ Options: r.options,
+ Progress: r.progress,
+ Catalog: r.catalog,
+ IssuesClient: r.issuesClient,
+ RateLimiter: r.rateLimiter,
+ Interactsh: r.interactsh,
+ ProjectFile: r.projectFile,
+ Browser: r.browser,
+ Colorizer: r.colorizer,
+ ResumeCfg: r.resumeCfg,
+ ExcludeMatchers: excludematchers.New(r.options.ExcludeMatchers),
+ InputHelper: input.NewHelper(),
+ TemporaryDirectory: r.tmpDir,
+ Parser: r.parser,
+ FuzzParamsFrequency: fuzzFreqCache,
}
if config.DefaultConfig.IsDebugArgEnabled(config.DebugExportURLPattern) {
@@ -619,6 +625,7 @@ func (r *Runner) RunEnumeration() error {
if executorOpts.InputHelper != nil {
_ = executorOpts.InputHelper.Close()
}
+ r.fuzzFrequencyCache.Close()
// todo: error propagation without canonical straight error check is required by cloud?
// use safe dereferencing to avoid potential panics in case of previous unchecked errors
diff --git a/pkg/fuzz/component/path.go b/pkg/fuzz/component/path.go
index 9d2e758025..b9aebd61af 100644
--- a/pkg/fuzz/component/path.go
+++ b/pkg/fuzz/component/path.go
@@ -2,10 +2,12 @@ package component
import (
"context"
+ "strconv"
+ "strings"
- "github.com/pkg/errors"
"github.com/projectdiscovery/nuclei/v3/pkg/fuzz/dataformat"
"github.com/projectdiscovery/retryablehttp-go"
+ urlutil "github.com/projectdiscovery/utils/url"
)
// Path is a component for a request Path
@@ -31,13 +33,18 @@ func (q *Path) Name() string {
// parsed component
func (q *Path) Parse(req *retryablehttp.Request) (bool, error) {
q.req = req
- q.value = NewValue(req.URL.Path)
+ q.value = NewValue("")
- parsed, err := dataformat.Get(dataformat.RawDataFormat).Decode(q.value.String())
- if err != nil {
- return false, err
+ splitted := strings.Split(req.URL.Path, "/")
+ values := make(map[string]interface{})
+ for i := range splitted {
+ pathTillNow := strings.Join(splitted[:i+1], "/")
+ if pathTillNow == "" {
+ continue
+ }
+ values[strconv.Itoa(i)] = pathTillNow
}
- q.value.SetParsed(parsed, dataformat.RawDataFormat)
+ q.value.SetParsed(dataformat.KVMap(values), "")
return true, nil
}
@@ -56,7 +63,8 @@ func (q *Path) Iterate(callback func(key string, value interface{}) error) (err
// SetValue sets a value in the component
// for a key
func (q *Path) SetValue(key string, value string) error {
- if !q.value.SetParsedValue(key, value) {
+ escaped := urlutil.ParamEncode(value)
+ if !q.value.SetParsedValue(key, escaped) {
return ErrSetValue
}
return nil
@@ -73,13 +81,31 @@ func (q *Path) Delete(key string) error {
// Rebuild returns a new request with the
// component rebuilt
func (q *Path) Rebuild() (*retryablehttp.Request, error) {
- encoded, err := q.value.Encode()
- if err != nil {
- return nil, errors.Wrap(err, "could not encode query")
+ originalValues := make(map[string]interface{})
+ splitted := strings.Split(q.req.URL.Path, "/")
+ for i := range splitted {
+ pathTillNow := strings.Join(splitted[:i+1], "/")
+ if pathTillNow == "" {
+ continue
+ }
+ originalValues[strconv.Itoa(i)] = pathTillNow
+ }
+
+ originalPath := q.req.URL.Path
+ lengthSplitted := len(q.value.parsed.Map)
+ for i := lengthSplitted; i > 0; i-- {
+ key := strconv.Itoa(i)
+ original := originalValues[key].(string)
+ new := q.value.parsed.Map[key].(string)
+ originalPath = strings.Replace(originalPath, original, new, 1)
}
+
+ rebuiltPath := originalPath
+
+ // Clone the request and update the path
cloned := q.req.Clone(context.Background())
- if err := cloned.UpdateRelPath(encoded, true); err != nil {
- cloned.URL.RawPath = encoded
+ if err := cloned.UpdateRelPath(rebuiltPath, true); err != nil {
+ cloned.URL.RawPath = rebuiltPath
}
return cloned, nil
}
diff --git a/pkg/fuzz/component/path_test.go b/pkg/fuzz/component/path_test.go
index 859ffcde12..c47f81f4ff 100644
--- a/pkg/fuzz/component/path_test.go
+++ b/pkg/fuzz/component/path_test.go
@@ -28,10 +28,10 @@ func TestURLComponent(t *testing.T) {
return nil
})
- require.Equal(t, []string{"value"}, keys, "unexpected keys")
+ require.Equal(t, []string{"1"}, keys, "unexpected keys")
require.Equal(t, []string{"/testpath"}, values, "unexpected values")
- err = urlComponent.SetValue("value", "/newpath")
+ err = urlComponent.SetValue("1", "/newpath")
if err != nil {
t.Fatal(err)
}
@@ -40,7 +40,41 @@ func TestURLComponent(t *testing.T) {
if err != nil {
t.Fatal(err)
}
-
require.Equal(t, "/newpath", rebuilt.URL.Path, "unexpected URL path")
require.Equal(t, "https://example.com/newpath", rebuilt.URL.String(), "unexpected full URL")
}
+
+func TestURLComponent_NestedPaths(t *testing.T) {
+ path := NewPath()
+ req, err := retryablehttp.NewRequest(http.MethodGet, "https://example.com/user/753/profile", nil)
+ if err != nil {
+ t.Fatal(err)
+ }
+ found, err := path.Parse(req)
+ if err != nil {
+ t.Fatal(err)
+ }
+ if !found {
+ t.Fatal("expected path to be found")
+ }
+
+ isSet := false
+
+ _ = path.Iterate(func(key string, value interface{}) error {
+ if !isSet && value.(string) == "/user/753" {
+ isSet = true
+ if setErr := path.SetValue(key, "/user/753'"); setErr != nil {
+ t.Fatal(setErr)
+ }
+ }
+ return nil
+ })
+
+ newReq, err := path.Rebuild()
+ if err != nil {
+ t.Fatal(err)
+ }
+ if newReq.URL.Path != "/user/753'/profile" {
+ t.Fatal("expected path to be modified")
+ }
+}
diff --git a/pkg/fuzz/component/query.go b/pkg/fuzz/component/query.go
index 3dc07e6cf1..571161ee13 100644
--- a/pkg/fuzz/component/query.go
+++ b/pkg/fuzz/component/query.go
@@ -61,6 +61,7 @@ func (q *Query) Iterate(callback func(key string, value interface{}) error) (err
// SetValue sets a value in the component
// for a key
func (q *Query) SetValue(key string, value string) error {
+ // Is this safe?
if !q.value.SetParsedValue(key, value) {
return ErrSetValue
}
diff --git a/pkg/fuzz/execute.go b/pkg/fuzz/execute.go
index bd1414638e..4f33ba6ddc 100644
--- a/pkg/fuzz/execute.go
+++ b/pkg/fuzz/execute.go
@@ -1,6 +1,7 @@
package fuzz
import (
+ "encoding/json"
"fmt"
"io"
"regexp"
@@ -15,6 +16,7 @@ import (
"github.com/projectdiscovery/nuclei/v3/pkg/protocols/common/generators"
"github.com/projectdiscovery/retryablehttp-go"
errorutil "github.com/projectdiscovery/utils/errors"
+ sliceutil "github.com/projectdiscovery/utils/slice"
urlutil "github.com/projectdiscovery/utils/url"
)
@@ -45,6 +47,8 @@ type ExecuteRuleInput struct {
Values map[string]interface{}
// BaseRequest is the base http request for fuzzing rule
BaseRequest *retryablehttp.Request
+ // DisplayFuzzPoints is a flag to display fuzz points
+ DisplayFuzzPoints bool
}
// GeneratedRequest is a single generated request for rule
@@ -76,8 +80,9 @@ func (rule *Rule) Execute(input *ExecuteRuleInput) (err error) {
var finalComponentList []component.Component
// match rule part with component name
+ displayDebugFuzzPoints := make(map[string]map[string]string)
for _, componentName := range component.Components {
- if rule.partType != requestPartType && rule.Part != componentName {
+ if !(rule.Part == componentName || sliceutil.Contains(rule.Parts, componentName) || rule.partType == requestPartType) {
continue
}
component := component.New(componentName)
@@ -89,12 +94,25 @@ func (rule *Rule) Execute(input *ExecuteRuleInput) (err error) {
if !discovered {
continue
}
+
// check rule applicable on this component
if !rule.checkRuleApplicableOnComponent(component) {
continue
}
+ // Debugging display for fuzz points
+ if input.DisplayFuzzPoints {
+ displayDebugFuzzPoints[componentName] = make(map[string]string)
+ _ = component.Iterate(func(key string, value interface{}) error {
+ displayDebugFuzzPoints[componentName][key] = fmt.Sprintf("%v", value)
+ return nil
+ })
+ }
finalComponentList = append(finalComponentList, component)
}
+ if len(displayDebugFuzzPoints) > 0 {
+ marshalled, _ := json.MarshalIndent(displayDebugFuzzPoints, "", " ")
+ gologger.Info().Msgf("[%s] Fuzz points for %s [%s]\n%s\n", rule.options.TemplateID, input.Input.MetaInput.Input, input.BaseRequest.Method, string(marshalled))
+ }
if len(finalComponentList) == 0 {
return ErrRuleNotApplicable.Msgf("no component matched on this rule")
@@ -225,7 +243,7 @@ func (rule *Rule) executeRuleValues(input *ExecuteRuleInput, ruleComponent compo
if err != nil {
return err
}
- if gotErr := rule.execWithInput(input, req, input.InteractURLs, ruleComponent, ""); gotErr != nil {
+ if gotErr := rule.execWithInput(input, req, input.InteractURLs, ruleComponent, "", ""); gotErr != nil {
return gotErr
}
}
@@ -261,8 +279,9 @@ func (rule *Rule) Compile(generator *generators.PayloadGenerator, options *proto
} else {
rule.partType = valueType
}
- } else {
- rule.partType = queryPartType
+ }
+ if rule.Part == "" && len(rule.Parts) == 0 {
+ return errors.Errorf("no part specified for rule")
}
if rule.Type != "" {
diff --git a/pkg/fuzz/frequency/tracker.go b/pkg/fuzz/frequency/tracker.go
new file mode 100644
index 0000000000..063912b4c0
--- /dev/null
+++ b/pkg/fuzz/frequency/tracker.go
@@ -0,0 +1,152 @@
+package frequency
+
+import (
+ "net"
+ "net/url"
+ "os"
+ "strings"
+ "sync"
+ "sync/atomic"
+
+ "github.com/bluele/gcache"
+ "github.com/projectdiscovery/gologger"
+)
+
+// Tracker implements a frequency tracker for a given input
+// which is used to determine uninteresting input parameters
+// which are not that interesting from fuzzing perspective for a template
+// and target combination.
+//
+// This is used to reduce the number of requests made during fuzzing
+// for parameters that are less likely to give results for a rule.
+type Tracker struct {
+ frequencies gcache.Cache
+ paramOccurenceThreshold int
+
+ isDebug bool
+}
+
+const (
+ DefaultMaxTrackCount = 10000
+ DefaultParamOccurenceThreshold = 10
+)
+
+type cacheItem struct {
+ errors atomic.Int32
+ sync.Once
+}
+
+// New creates a new frequency tracker with a given maximum
+// number of params to track in LRU fashion with a max error threshold
+func New(maxTrackCount, paramOccurenceThreshold int) *Tracker {
+ gc := gcache.New(maxTrackCount).ARC().Build()
+
+ var isDebug bool
+ if os.Getenv("FREQ_DEBUG") != "" {
+ isDebug = true
+ }
+ return &Tracker{
+ isDebug: isDebug,
+ frequencies: gc,
+ paramOccurenceThreshold: paramOccurenceThreshold,
+ }
+}
+
+func (t *Tracker) Close() {
+ t.frequencies.Purge()
+}
+
+// MarkParameter marks a parameter as frequently occuring once.
+//
+// The logic requires a parameter to be marked as frequently occuring
+// multiple times before it's considered as frequently occuring.
+func (t *Tracker) MarkParameter(parameter, target, template string) {
+ normalizedTarget := normalizeTarget(target)
+ key := getFrequencyKey(parameter, normalizedTarget, template)
+
+ if t.isDebug {
+ gologger.Verbose().Msgf("[%s] Marking %s as found uninteresting", template, key)
+ }
+
+ existingCacheItem, err := t.frequencies.GetIFPresent(key)
+ if err != nil || existingCacheItem == nil {
+ newItem := &cacheItem{errors: atomic.Int32{}}
+ newItem.errors.Store(1)
+ _ = t.frequencies.Set(key, newItem)
+ return
+ }
+ existingCacheItemValue := existingCacheItem.(*cacheItem)
+ existingCacheItemValue.errors.Add(1)
+
+ _ = t.frequencies.Set(key, existingCacheItemValue)
+}
+
+// IsParameterFrequent checks if a parameter is frequently occuring
+// in the input with no much results.
+func (t *Tracker) IsParameterFrequent(parameter, target, template string) bool {
+ normalizedTarget := normalizeTarget(target)
+ key := getFrequencyKey(parameter, normalizedTarget, template)
+
+ if t.isDebug {
+ gologger.Verbose().Msgf("[%s] Checking if %s is frequently found uninteresting", template, key)
+ }
+
+ existingCacheItem, err := t.frequencies.GetIFPresent(key)
+ if err != nil {
+ return false
+ }
+ existingCacheItemValue := existingCacheItem.(*cacheItem)
+
+ if existingCacheItemValue.errors.Load() >= int32(t.paramOccurenceThreshold) {
+ existingCacheItemValue.Do(func() {
+ gologger.Verbose().Msgf("[%s] Skipped %s from parameter for %s as found uninteresting %d times", template, parameter, target, existingCacheItemValue.errors.Load())
+ })
+ return true
+ }
+ return false
+}
+
+// UnmarkParameter unmarks a parameter as frequently occuring. This carries
+// more weight and resets the frequency counter for the parameter causing
+// it to be checked again. This is done when results are found.
+func (t *Tracker) UnmarkParameter(parameter, target, template string) {
+ normalizedTarget := normalizeTarget(target)
+ key := getFrequencyKey(parameter, normalizedTarget, template)
+
+ if t.isDebug {
+ gologger.Verbose().Msgf("[%s] Unmarking %s as frequently found uninteresting", template, key)
+ }
+
+ _ = t.frequencies.Remove(key)
+}
+
+func getFrequencyKey(parameter, target, template string) string {
+ var sb strings.Builder
+ sb.WriteString(target)
+ sb.WriteString(":")
+ sb.WriteString(template)
+ sb.WriteString(":")
+ sb.WriteString(parameter)
+ str := sb.String()
+ return str
+}
+
+func normalizeTarget(value string) string {
+ finalValue := value
+ if strings.HasPrefix(value, "http") {
+ if parsed, err := url.Parse(value); err == nil {
+ hostname := parsed.Host
+ finalPort := parsed.Port()
+ if finalPort == "" {
+ if parsed.Scheme == "https" {
+ finalPort = "443"
+ } else {
+ finalPort = "80"
+ }
+ hostname = net.JoinHostPort(parsed.Host, finalPort)
+ }
+ finalValue = hostname
+ }
+ }
+ return finalValue
+}
diff --git a/pkg/fuzz/fuzz.go b/pkg/fuzz/fuzz.go
index c70bc3b2cc..b2109efc85 100644
--- a/pkg/fuzz/fuzz.go
+++ b/pkg/fuzz/fuzz.go
@@ -24,12 +24,27 @@ type Rule struct {
ruleType ruleType
// description: |
// Part is the part of request to fuzz.
- //
- // query fuzzes the query part of url. More parts will be added later.
// values:
// - "query"
+ // - "header"
+ // - "path"
+ // - "body"
+ // - "cookie"
+ // - "request"
Part string `yaml:"part,omitempty" json:"part,omitempty" jsonschema:"title=part of rule,description=Part of request rule to fuzz,enum=query,enum=header,enum=path,enum=body,enum=cookie,enum=request"`
partType partType
+ // description: |
+ // Parts is the list of parts to fuzz. If multiple parts need to be
+ // defined while excluding some, this should be used instead of singular part.
+ // values:
+ // - "query"
+ // - "header"
+ // - "path"
+ // - "body"
+ // - "cookie"
+ // - "request"
+ Parts []string `yaml:"parts,omitempty" json:"parts,omitempty" jsonschema:"title=parts of rule,description=Part of request rule to fuzz,enum=query,enum=header,enum=path,enum=body,enum=cookie,enum=request"`
+
// description: |
// Mode is the mode of fuzzing to perform.
//
diff --git a/pkg/fuzz/fuzz_test.go b/pkg/fuzz/fuzz_test.go
index dc0771259b..6ef2e39b08 100644
--- a/pkg/fuzz/fuzz_test.go
+++ b/pkg/fuzz/fuzz_test.go
@@ -7,7 +7,9 @@ import (
)
func TestRuleMatchKeyOrValue(t *testing.T) {
- rule := &Rule{}
+ rule := &Rule{
+ Part: "query",
+ }
err := rule.Compile(nil, nil)
require.NoError(t, err, "could not compile rule")
@@ -15,7 +17,7 @@ func TestRuleMatchKeyOrValue(t *testing.T) {
require.True(t, result, "could not get correct result")
t.Run("key", func(t *testing.T) {
- rule := &Rule{Keys: []string{"url"}}
+ rule := &Rule{Keys: []string{"url"}, Part: "query"}
err := rule.Compile(nil, nil)
require.NoError(t, err, "could not compile rule")
@@ -25,7 +27,7 @@ func TestRuleMatchKeyOrValue(t *testing.T) {
require.False(t, result, "could not get correct result")
})
t.Run("value", func(t *testing.T) {
- rule := &Rule{ValuesRegex: []string{`https?:\/\/?([-a-zA-Z0-9@:%._\+~#=]{2,256}\.[a-z]{2,6}\b)*(\/[\/\d\w\.-]*)*(?:[\?])*(.+)*`}}
+ rule := &Rule{ValuesRegex: []string{`https?:\/\/?([-a-zA-Z0-9@:%._\+~#=]{2,256}\.[a-z]{2,6}\b)*(\/[\/\d\w\.-]*)*(?:[\?])*(.+)*`}, Part: "query"}
err := rule.Compile(nil, nil)
require.NoError(t, err, "could not compile rule")
diff --git a/pkg/fuzz/parts.go b/pkg/fuzz/parts.go
index 4c01135f66..6ab1643296 100644
--- a/pkg/fuzz/parts.go
+++ b/pkg/fuzz/parts.go
@@ -2,6 +2,7 @@ package fuzz
import (
"io"
+ "strconv"
"strings"
"github.com/projectdiscovery/nuclei/v3/pkg/fuzz/component"
@@ -9,6 +10,7 @@ import (
"github.com/projectdiscovery/nuclei/v3/pkg/protocols/common/generators"
"github.com/projectdiscovery/nuclei/v3/pkg/types"
"github.com/projectdiscovery/retryablehttp-go"
+ sliceutil "github.com/projectdiscovery/utils/slice"
)
// executePartRule executes part rules based on type
@@ -18,7 +20,7 @@ func (rule *Rule) executePartRule(input *ExecuteRuleInput, payload ValueOrKeyVal
// checkRuleApplicableOnComponent checks if a rule is applicable on given component
func (rule *Rule) checkRuleApplicableOnComponent(component component.Component) bool {
- if rule.Part != component.Name() {
+ if rule.Part != component.Name() && !sliceutil.Contains(rule.Parts, component.Name()) && rule.partType != requestPartType {
return false
}
foundAny := false
@@ -68,7 +70,7 @@ func (rule *Rule) executePartComponentOnValues(input *ExecuteRuleInput, payloadS
return err
}
- if qerr := rule.execWithInput(input, req, input.InteractURLs, ruleComponent, key); qerr != nil {
+ if qerr := rule.execWithInput(input, req, input.InteractURLs, ruleComponent, key, valueStr); qerr != nil {
return qerr
}
// fmt.Printf("executed with value: %s\n", evaluated)
@@ -90,7 +92,7 @@ func (rule *Rule) executePartComponentOnValues(input *ExecuteRuleInput, payloadS
if err != nil {
return err
}
- if qerr := rule.execWithInput(input, req, input.InteractURLs, ruleComponent, ""); qerr != nil {
+ if qerr := rule.execWithInput(input, req, input.InteractURLs, ruleComponent, "", ""); qerr != nil {
err = qerr
return err
}
@@ -125,7 +127,7 @@ func (rule *Rule) executePartComponentOnKV(input *ExecuteRuleInput, payload Valu
return err
}
- if qerr := rule.execWithInput(input, req, input.InteractURLs, ruleComponent, key); qerr != nil {
+ if qerr := rule.execWithInput(input, req, input.InteractURLs, ruleComponent, key, value); qerr != nil {
return err
}
@@ -144,7 +146,23 @@ func (rule *Rule) executePartComponentOnKV(input *ExecuteRuleInput, payload Valu
}
// execWithInput executes a rule with input via callback
-func (rule *Rule) execWithInput(input *ExecuteRuleInput, httpReq *retryablehttp.Request, interactURLs []string, component component.Component, parameter string) error {
+func (rule *Rule) execWithInput(input *ExecuteRuleInput, httpReq *retryablehttp.Request, interactURLs []string, component component.Component, parameter, parameterValue string) error {
+ // If the parameter is a number, replace it with the parameter value
+ // or if the parameter is empty and the parameter value is not empty
+ // replace it with the parameter value
+ if _, err := strconv.Atoi(parameter); err == nil || (parameter == "" && parameterValue != "") {
+ parameter = parameterValue
+ }
+ // If the parameter is frequent, skip it if the option is enabled
+ if rule.options.FuzzParamsFrequency != nil {
+ if rule.options.FuzzParamsFrequency.IsParameterFrequent(
+ parameter,
+ httpReq.URL.String(),
+ rule.options.TemplateID,
+ ) {
+ return nil
+ }
+ }
request := GeneratedRequest{
Request: httpReq,
InteractURLs: interactURLs,
diff --git a/pkg/protocols/common/generators/generators.go b/pkg/protocols/common/generators/generators.go
index b99de24c0b..5711b445f5 100644
--- a/pkg/protocols/common/generators/generators.go
+++ b/pkg/protocols/common/generators/generators.go
@@ -25,8 +25,19 @@ func New(payloads map[string]interface{}, attackType AttackType, templatePath st
// Resolve payload paths if they are files.
payloadsFinal := make(map[string]interface{})
- for name, payload := range payloads {
- payloadsFinal[name] = payload
+ for payloadName, v := range payloads {
+ switch value := v.(type) {
+ case map[interface{}]interface{}:
+ values, err := parsePayloadsWithAggression(payloadName, value, opts.FuzzAggressionLevel)
+ if err != nil {
+ return nil, errors.Wrap(err, "could not parse payloads with aggression")
+ }
+ for k, v := range values {
+ payloadsFinal[k] = v
+ }
+ default:
+ payloadsFinal[payloadName] = v
+ }
}
generator := &PayloadGenerator{catalog: catalog, options: opts}
@@ -57,6 +68,60 @@ func New(payloads map[string]interface{}, attackType AttackType, templatePath st
return generator, nil
}
+type aggressionLevelToPayloads struct {
+ Low []interface{}
+ Medium []interface{}
+ High []interface{}
+}
+
+// parsePayloadsWithAggression parses the payloads with the aggression level
+//
+// Three agression are supported -
+// - low
+// - medium
+// - high
+//
+// low is the default level. If medium is specified, all templates from
+// low and medium are executed. Similarly with high, including all templates
+// from low, medium, high.
+func parsePayloadsWithAggression(name string, v map[interface{}]interface{}, agression string) (map[string]interface{}, error) {
+ payloadsLevels := &aggressionLevelToPayloads{}
+
+ for k, v := range v {
+ if _, ok := v.([]interface{}); !ok {
+ return nil, errors.Errorf("only lists are supported for aggression levels payloads")
+ }
+ var ok bool
+ switch k {
+ case "low":
+ payloadsLevels.Low, ok = v.([]interface{})
+ case "medium":
+ payloadsLevels.Medium, ok = v.([]interface{})
+ case "high":
+ payloadsLevels.High, ok = v.([]interface{})
+ default:
+ return nil, errors.Errorf("invalid aggression level %s specified for %s", k, name)
+ }
+ if !ok {
+ return nil, errors.Errorf("invalid aggression level %s specified for %s", k, name)
+ }
+ }
+
+ payloads := make(map[string]interface{})
+ switch agression {
+ case "low":
+ payloads[name] = payloadsLevels.Low
+ case "medium":
+ payloads[name] = append(payloadsLevels.Low, payloadsLevels.Medium...)
+ case "high":
+ payloads[name] = append(payloadsLevels.Low, payloadsLevels.Medium...)
+ payloads[name] = append(payloads[name].([]interface{}), payloadsLevels.High...)
+ default:
+ return nil, errors.Errorf("invalid aggression level %s specified for %s", agression, name)
+ }
+ return payloads, nil
+}
+
// Iterator is a single instance of an iterator for a generator structure
type Iterator struct {
Type AttackType
diff --git a/pkg/protocols/common/generators/generators_test.go b/pkg/protocols/common/generators/generators_test.go
index a55e8e51ee..c478995525 100644
--- a/pkg/protocols/common/generators/generators_test.go
+++ b/pkg/protocols/common/generators/generators_test.go
@@ -1,9 +1,11 @@
package generators
import (
+ "strings"
"testing"
"github.com/stretchr/testify/require"
+ "gopkg.in/yaml.v2"
"github.com/projectdiscovery/nuclei/v3/pkg/catalog/disk"
"github.com/projectdiscovery/nuclei/v3/pkg/types"
@@ -90,3 +92,49 @@ func getOptions(allowLocalFileAccess bool) *types.Options {
opts.AllowLocalFileAccess = allowLocalFileAccess
return opts
}
+
+func TestParsePayloadsWithAggression(t *testing.T) {
+ testPayload := `linux_path:
+ low:
+ - /etc/passwd
+ medium:
+ - ../etc/passwd
+ - ../../etc/passwd
+ high:
+ - ../../../etc/passwd
+ - ../../../../etc/passwd
+ - ../../../../../etc/passwd`
+
+ var payloads map[string]interface{}
+ err := yaml.NewDecoder(strings.NewReader(testPayload)).Decode(&payloads)
+ require.Nil(t, err, "could not unmarshal yaml")
+
+ aggressionsToValues := map[string][]string{
+ "low": {
+ "/etc/passwd",
+ },
+ "medium": {
+ "/etc/passwd",
+ "../etc/passwd",
+ "../../etc/passwd",
+ },
+ "high": {
+ "/etc/passwd",
+ "../etc/passwd",
+ "../../etc/passwd",
+ "../../../etc/passwd",
+ "../../../../etc/passwd",
+ "../../../../../etc/passwd",
+ },
+ }
+
+ for k, v := range payloads {
+ for aggression, values := range aggressionsToValues {
+ parsed, err := parsePayloadsWithAggression(k, v.(map[interface{}]interface{}), aggression)
+ require.Nil(t, err, "could not parse payloads with aggression")
+
+ gotValues := parsed[k].([]interface{})
+ require.Equal(t, len(values), len(gotValues), "could not get correct number of values")
+ }
+ }
+}
diff --git a/pkg/protocols/common/interactsh/interactsh.go b/pkg/protocols/common/interactsh/interactsh.go
index c156164805..da59f10fb3 100644
--- a/pkg/protocols/common/interactsh/interactsh.go
+++ b/pkg/protocols/common/interactsh/interactsh.go
@@ -21,6 +21,7 @@ import (
"github.com/projectdiscovery/nuclei/v3/pkg/output"
"github.com/projectdiscovery/nuclei/v3/pkg/protocols/common/helpers/responsehighlighter"
"github.com/projectdiscovery/nuclei/v3/pkg/protocols/common/helpers/writer"
+ "github.com/projectdiscovery/retryablehttp-go"
errorutil "github.com/projectdiscovery/utils/errors"
stringsutil "github.com/projectdiscovery/utils/strings"
)
@@ -180,6 +181,14 @@ func (c *Client) processInteractionForRequest(interaction *server.Interaction, d
gologger.DefaultLogger.Print().Msgf("[Interactsh]: got result %v and status %v after processing interaction", result, matched)
}
+ if c.options.FuzzParamsFrequency != nil {
+ if !matched {
+ c.options.FuzzParamsFrequency.MarkParameter(data.Parameter, data.Request.URL.String(), data.Operators.TemplateID)
+ } else {
+ c.options.FuzzParamsFrequency.UnmarkParameter(data.Parameter, data.Request.URL.String(), data.Operators.TemplateID)
+ }
+ }
+
// if we don't match, return
if !matched || result == nil {
return false
@@ -320,6 +329,9 @@ type RequestData struct {
Operators *operators.Operators
MatchFunc operators.MatchFunc
ExtractFunc operators.ExtractFunc
+
+ Parameter string
+ Request *retryablehttp.Request
}
// RequestEvent is the event for a network request sent by nuclei.
diff --git a/pkg/protocols/common/interactsh/options.go b/pkg/protocols/common/interactsh/options.go
index c2ae250291..ca3dd459c9 100644
--- a/pkg/protocols/common/interactsh/options.go
+++ b/pkg/protocols/common/interactsh/options.go
@@ -4,6 +4,7 @@ import (
"time"
"github.com/projectdiscovery/interactsh/pkg/client"
+ "github.com/projectdiscovery/nuclei/v3/pkg/fuzz/frequency"
"github.com/projectdiscovery/nuclei/v3/pkg/output"
"github.com/projectdiscovery/nuclei/v3/pkg/progress"
"github.com/projectdiscovery/nuclei/v3/pkg/reporting"
@@ -46,8 +47,9 @@ type Options struct {
// NoColor disables printing colors for matches
NoColor bool
- StopAtFirstMatch bool
- HTTPClient *retryablehttp.Client
+ FuzzParamsFrequency *frequency.Tracker
+ StopAtFirstMatch bool
+ HTTPClient *retryablehttp.Client
}
// DefaultOptions returns the default options for interactsh client
diff --git a/pkg/protocols/http/request_fuzz.go b/pkg/protocols/http/request_fuzz.go
index 8d980b4bf3..49df3c0632 100644
--- a/pkg/protocols/http/request_fuzz.go
+++ b/pkg/protocols/http/request_fuzz.go
@@ -120,7 +120,8 @@ func (request *Request) executeAllFuzzingRules(input *contextargs.Context, value
}
err := rule.Execute(&fuzz.ExecuteRuleInput{
- Input: input,
+ Input: input,
+ DisplayFuzzPoints: request.options.Options.DisplayFuzzPoints,
Callback: func(gr fuzz.GeneratedRequest) bool {
select {
case <-input.Context().Done():
@@ -139,6 +140,7 @@ func (request *Request) executeAllFuzzingRules(input *contextargs.Context, value
continue
}
if fuzz.IsErrRuleNotApplicable(err) {
+ gologger.Verbose().Msgf("[%s] fuzz: rule not applicable : %s\n", request.options.TemplateID, err)
continue
}
if err == types.ErrNoMoreRequests {
@@ -176,6 +178,7 @@ func (request *Request) executeGeneratedFuzzingRequest(gr fuzz.GeneratedRequest,
result.FuzzingPosition = gr.Component.Name()
}
+ setInteractshCallback := false
if hasInteractMarkers && hasInteractMatchers && request.options.Interactsh != nil {
requestData := &interactsh.RequestData{
MakeResultFunc: request.MakeResultEvent,
@@ -183,7 +186,10 @@ func (request *Request) executeGeneratedFuzzingRequest(gr fuzz.GeneratedRequest,
Operators: request.CompiledOperators,
MatchFunc: request.Match,
ExtractFunc: request.Extract,
+ Parameter: gr.Parameter,
+ Request: gr.Request,
}
+ setInteractshCallback = true
request.options.Interactsh.RequestEvent(gr.InteractURLs, requestData)
gotMatches = request.options.Interactsh.AlreadyMatched(requestData)
} else {
@@ -193,6 +199,13 @@ func (request *Request) executeGeneratedFuzzingRequest(gr fuzz.GeneratedRequest,
if event.OperatorsResult != nil {
gotMatches = event.OperatorsResult.Matched
}
+ if request.options.FuzzParamsFrequency != nil && !setInteractshCallback {
+ if !gotMatches {
+ request.options.FuzzParamsFrequency.MarkParameter(gr.Parameter, gr.Request.URL.String(), request.options.TemplateID)
+ } else {
+ request.options.FuzzParamsFrequency.UnmarkParameter(gr.Parameter, gr.Request.URL.String(), request.options.TemplateID)
+ }
+ }
}, 0)
// If a variable is unresolved, skip all further requests
if errors.Is(requestErr, ErrMissingVars) {
diff --git a/pkg/protocols/protocols.go b/pkg/protocols/protocols.go
index 6b4904c8d8..68881fec5c 100644
--- a/pkg/protocols/protocols.go
+++ b/pkg/protocols/protocols.go
@@ -13,6 +13,7 @@ import (
"github.com/projectdiscovery/nuclei/v3/pkg/authprovider"
"github.com/projectdiscovery/nuclei/v3/pkg/catalog"
+ "github.com/projectdiscovery/nuclei/v3/pkg/fuzz/frequency"
"github.com/projectdiscovery/nuclei/v3/pkg/input"
"github.com/projectdiscovery/nuclei/v3/pkg/js/compiler"
"github.com/projectdiscovery/nuclei/v3/pkg/loader/parser"
@@ -92,6 +93,8 @@ type ExecutorOptions struct {
ExcludeMatchers *excludematchers.ExcludeMatchers
// InputHelper is a helper for input normalization
InputHelper *input.Helper
+ // FuzzParamsFrequency is a cache for parameter frequency
+ FuzzParamsFrequency *frequency.Tracker
Operators []*operators.Operators // only used by offlinehttp module
diff --git a/pkg/testutils/fuzzplayground/server.go b/pkg/testutils/fuzzplayground/server.go
index 04f00a99df..af42552cbf 100644
--- a/pkg/testutils/fuzzplayground/server.go
+++ b/pkg/testutils/fuzzplayground/server.go
@@ -27,6 +27,7 @@ func GetPlaygroundServer() *echo.Echo {
e.GET("/request", requestHandler)
e.GET("/email", emailHandler)
e.GET("/permissions", permissionsHandler)
+
e.GET("/blog/post", numIdorHandler) // for num based idors like ?id=44
e.POST("/reset-password", resetPasswordHandler)
e.GET("/host-header-lab", hostHeaderLabHandler)
@@ -47,13 +48,20 @@ var bodyTemplate = `
func indexHandler(ctx echo.Context) error {
return ctx.HTML(200, fmt.Sprintf(bodyTemplate, `Fuzzing Playground
-
+
`))
}
diff --git a/pkg/types/types.go b/pkg/types/types.go
index 0c31769ee1..28f98f9198 100644
--- a/pkg/types/types.go
+++ b/pkg/types/types.go
@@ -363,6 +363,12 @@ type Options struct {
FuzzingMode string
// TlsImpersonate enables TLS impersonation
TlsImpersonate bool
+ // DisplayFuzzPoints enables display of fuzz points for fuzzing
+ DisplayFuzzPoints bool
+ // FuzzAggressionLevel is the level of fuzzing aggression (low, medium, high.)
+ FuzzAggressionLevel string
+ // FuzzParamFrequency is the frequency of fuzzing parameters
+ FuzzParamFrequency int
// CodeTemplateSignaturePublicKey is the custom public key used to verify the template signature (algorithm is automatically inferred from the length)
CodeTemplateSignaturePublicKey string
// CodeTemplateSignatureAlgorithm specifies the sign algorithm (rsa, ecdsa)
From 1c355c54ad42c19439e7d4ce7d54c4c3bca1c5bc Mon Sep 17 00:00:00 2001
From: GitHub Action
Date: Mon, 10 Jun 2024 23:14:52 +0000
Subject: [PATCH 044/283] Auto Generate Syntax Docs + JSONSchema [Mon Jun 10
23:14:52 UTC 2024] :robot:
---
SYNTAX-REFERENCE.md | 39 ++++++++++++++++-
nuclei-jsonschema.json | 16 +++++++
pkg/templates/templates_doc.go | 76 +++++++++++++++++++++-------------
3 files changed, 101 insertions(+), 30 deletions(-)
diff --git a/SYNTAX-REFERENCE.md b/SYNTAX-REFERENCE.md
index 7fa62c5f44..0db5711f5a 100755
--- a/SYNTAX-REFERENCE.md
+++ b/SYNTAX-REFERENCE.md
@@ -1800,13 +1800,50 @@ Valid values:
Part is the part of request to fuzz.
-query fuzzes the query part of url. More parts will be added later.
+
+Valid values:
+
+
+ - query
+
+ - header
+
+ - path
+
+ - body
+
+ - cookie
+
+ - request
+
+
+
+
+
+
+parts
[]string
+
+
+
+
+Parts is the list of parts to fuzz. If multiple parts need to be
+defined while excluding some, this should be used instead of singular part.
Valid values:
- query
+
+ - header
+
+ - path
+
+ - body
+
+ - cookie
+
+ - request
diff --git a/nuclei-jsonschema.json b/nuclei-jsonschema.json
index 46e1ab986d..61bdc68ff8 100644
--- a/nuclei-jsonschema.json
+++ b/nuclei-jsonschema.json
@@ -457,6 +457,22 @@
"title": "part of rule",
"description": "Part of request rule to fuzz"
},
+ "parts": {
+ "items": {
+ "type": "string",
+ "enum": [
+ "query",
+ "header",
+ "path",
+ "body",
+ "cookie",
+ "request"
+ ]
+ },
+ "type": "array",
+ "title": "parts of rule",
+ "description": "Part of request rule to fuzz"
+ },
"mode": {
"type": "string",
"enum": [
diff --git a/pkg/templates/templates_doc.go b/pkg/templates/templates_doc.go
index 42fa25b947..536b80ad7f 100644
--- a/pkg/templates/templates_doc.go
+++ b/pkg/templates/templates_doc.go
@@ -747,7 +747,7 @@ func init() {
FieldName: "fuzzing",
},
}
- FUZZRuleDoc.Fields = make([]encoder.Doc, 8)
+ FUZZRuleDoc.Fields = make([]encoder.Doc, 9)
FUZZRuleDoc.Fields[0].Name = "type"
FUZZRuleDoc.Fields[0].Type = "string"
FUZZRuleDoc.Fields[0].Note = ""
@@ -762,51 +762,69 @@ func init() {
FUZZRuleDoc.Fields[1].Name = "part"
FUZZRuleDoc.Fields[1].Type = "string"
FUZZRuleDoc.Fields[1].Note = ""
- FUZZRuleDoc.Fields[1].Description = "Part is the part of request to fuzz.\n\nquery fuzzes the query part of url. More parts will be added later."
+ FUZZRuleDoc.Fields[1].Description = "Part is the part of request to fuzz."
FUZZRuleDoc.Fields[1].Comments[encoder.LineComment] = "Part is the part of request to fuzz."
FUZZRuleDoc.Fields[1].Values = []string{
"query",
+ "header",
+ "path",
+ "body",
+ "cookie",
+ "request",
}
- FUZZRuleDoc.Fields[2].Name = "mode"
- FUZZRuleDoc.Fields[2].Type = "string"
+ FUZZRuleDoc.Fields[2].Name = "parts"
+ FUZZRuleDoc.Fields[2].Type = "[]string"
FUZZRuleDoc.Fields[2].Note = ""
- FUZZRuleDoc.Fields[2].Description = "Mode is the mode of fuzzing to perform.\n\nsingle fuzzes one value at a time. multiple fuzzes all values at same time."
- FUZZRuleDoc.Fields[2].Comments[encoder.LineComment] = "Mode is the mode of fuzzing to perform."
+ FUZZRuleDoc.Fields[2].Description = "Parts is the list of parts to fuzz. If multiple parts need to be\ndefined while excluding some, this should be used instead of singular part."
+ FUZZRuleDoc.Fields[2].Comments[encoder.LineComment] = "Parts is the list of parts to fuzz. If multiple parts need to be"
FUZZRuleDoc.Fields[2].Values = []string{
+ "query",
+ "header",
+ "path",
+ "body",
+ "cookie",
+ "request",
+ }
+ FUZZRuleDoc.Fields[3].Name = "mode"
+ FUZZRuleDoc.Fields[3].Type = "string"
+ FUZZRuleDoc.Fields[3].Note = ""
+ FUZZRuleDoc.Fields[3].Description = "Mode is the mode of fuzzing to perform.\n\nsingle fuzzes one value at a time. multiple fuzzes all values at same time."
+ FUZZRuleDoc.Fields[3].Comments[encoder.LineComment] = "Mode is the mode of fuzzing to perform."
+ FUZZRuleDoc.Fields[3].Values = []string{
"single",
"multiple",
}
- FUZZRuleDoc.Fields[3].Name = "keys"
- FUZZRuleDoc.Fields[3].Type = "[]string"
- FUZZRuleDoc.Fields[3].Note = ""
- FUZZRuleDoc.Fields[3].Description = "Keys is the optional list of key named parameters to fuzz."
- FUZZRuleDoc.Fields[3].Comments[encoder.LineComment] = "Keys is the optional list of key named parameters to fuzz."
-
- FUZZRuleDoc.Fields[3].AddExample("Examples of keys", []string{"url", "file", "host"})
- FUZZRuleDoc.Fields[4].Name = "keys-regex"
+ FUZZRuleDoc.Fields[4].Name = "keys"
FUZZRuleDoc.Fields[4].Type = "[]string"
FUZZRuleDoc.Fields[4].Note = ""
- FUZZRuleDoc.Fields[4].Description = "KeysRegex is the optional list of regex key parameters to fuzz."
- FUZZRuleDoc.Fields[4].Comments[encoder.LineComment] = "KeysRegex is the optional list of regex key parameters to fuzz."
+ FUZZRuleDoc.Fields[4].Description = "Keys is the optional list of key named parameters to fuzz."
+ FUZZRuleDoc.Fields[4].Comments[encoder.LineComment] = "Keys is the optional list of key named parameters to fuzz."
- FUZZRuleDoc.Fields[4].AddExample("Examples of key regex", []string{"url.*"})
- FUZZRuleDoc.Fields[5].Name = "values"
+ FUZZRuleDoc.Fields[4].AddExample("Examples of keys", []string{"url", "file", "host"})
+ FUZZRuleDoc.Fields[5].Name = "keys-regex"
FUZZRuleDoc.Fields[5].Type = "[]string"
FUZZRuleDoc.Fields[5].Note = ""
- FUZZRuleDoc.Fields[5].Description = "Values is the optional list of regex value parameters to fuzz."
- FUZZRuleDoc.Fields[5].Comments[encoder.LineComment] = "Values is the optional list of regex value parameters to fuzz."
+ FUZZRuleDoc.Fields[5].Description = "KeysRegex is the optional list of regex key parameters to fuzz."
+ FUZZRuleDoc.Fields[5].Comments[encoder.LineComment] = "KeysRegex is the optional list of regex key parameters to fuzz."
- FUZZRuleDoc.Fields[5].AddExample("Examples of value regex", []string{"https?://.*"})
- FUZZRuleDoc.Fields[6].Name = "fuzz"
- FUZZRuleDoc.Fields[6].Type = "SliceOrMapSlice"
+ FUZZRuleDoc.Fields[5].AddExample("Examples of key regex", []string{"url.*"})
+ FUZZRuleDoc.Fields[6].Name = "values"
+ FUZZRuleDoc.Fields[6].Type = "[]string"
FUZZRuleDoc.Fields[6].Note = ""
- FUZZRuleDoc.Fields[6].Description = "description: |\n Fuzz is the list of payloads to perform substitutions with.\n examples:\n - name: Examples of fuzz\n value: >\n []string{\"{{ssrf}}\", \"{{interactsh-url}}\", \"example-value\"}\n or\n x-header: 1\n x-header: 2"
- FUZZRuleDoc.Fields[6].Comments[encoder.LineComment] = " description: |"
- FUZZRuleDoc.Fields[7].Name = "replace-regex"
- FUZZRuleDoc.Fields[7].Type = "string"
+ FUZZRuleDoc.Fields[6].Description = "Values is the optional list of regex value parameters to fuzz."
+ FUZZRuleDoc.Fields[6].Comments[encoder.LineComment] = "Values is the optional list of regex value parameters to fuzz."
+
+ FUZZRuleDoc.Fields[6].AddExample("Examples of value regex", []string{"https?://.*"})
+ FUZZRuleDoc.Fields[7].Name = "fuzz"
+ FUZZRuleDoc.Fields[7].Type = "SliceOrMapSlice"
FUZZRuleDoc.Fields[7].Note = ""
- FUZZRuleDoc.Fields[7].Description = "replace-regex is regex for regex-replace rule type\nit is only required for replace-regex rule type"
- FUZZRuleDoc.Fields[7].Comments[encoder.LineComment] = "replace-regex is regex for regex-replace rule type"
+ FUZZRuleDoc.Fields[7].Description = "description: |\n Fuzz is the list of payloads to perform substitutions with.\n examples:\n - name: Examples of fuzz\n value: >\n []string{\"{{ssrf}}\", \"{{interactsh-url}}\", \"example-value\"}\n or\n x-header: 1\n x-header: 2"
+ FUZZRuleDoc.Fields[7].Comments[encoder.LineComment] = " description: |"
+ FUZZRuleDoc.Fields[8].Name = "replace-regex"
+ FUZZRuleDoc.Fields[8].Type = "string"
+ FUZZRuleDoc.Fields[8].Note = ""
+ FUZZRuleDoc.Fields[8].Description = "replace-regex is regex for regex-replace rule type\nit is only required for replace-regex rule type"
+ FUZZRuleDoc.Fields[8].Comments[encoder.LineComment] = "replace-regex is regex for regex-replace rule type"
SliceOrMapSliceDoc.Type = "SliceOrMapSlice"
SliceOrMapSliceDoc.Comments[encoder.LineComment] = ""
From 0b6ad3597f48b460495e21d9bd97b6258ca6b9bc Mon Sep 17 00:00:00 2001
From: sandeep <8293321+ehsandeep@users.noreply.github.com>
Date: Tue, 11 Jun 2024 04:48:05 +0530
Subject: [PATCH 045/283] readme update
---
README.md | 21 +++++++++++++++------
1 file changed, 15 insertions(+), 6 deletions(-)
diff --git a/README.md b/README.md
index 54242fd944..26b3d4a727 100644
--- a/README.md
+++ b/README.md
@@ -182,6 +182,8 @@ OUTPUT:
CONFIGURATIONS:
-config string path to the nuclei configuration file
+ -tp, -profile string template profile config file to run
+ -tpl, -profile-list list community template profiles
-fr, -follow-redirects enable following redirects for http templates
-fhr, -follow-host-redirects follow redirects on the same host
-mr, -max-redirects int max number of redirects to follow for http templates (default 10)
@@ -208,10 +210,12 @@ CONFIGURATIONS:
-i, -interface string network interface to use for network scan
-at, -attack-type string type of payload combinations to perform (batteringram,pitchfork,clusterbomb)
-sip, -source-ip string source ip address to use for network scan
- -rsr, -response-size-read int max response size to read in bytes (default 10485760)
+ -rsr, -response-size-read int max response size to read in bytes
-rss, -response-size-save int max response size to read in bytes (default 1048576)
+ -rrt, -response-read-timeout value response read timeout in seconds (default 5s)
-reset reset removes all nuclei configuration and data files (including nuclei-templates)
-tlsi, -tls-impersonate enable experimental client hello (ja3) tls randomization
+ -hae, -http-api-endpoint string experimental http api endpoint
INTERACTSH:
-iserver, -interactsh-server string interactsh server url for self-hosted instance (default: oast.pro,oast.live,oast.site,oast.online,oast.fun,oast.me)
@@ -223,10 +227,13 @@ INTERACTSH:
-ni, -no-interactsh disable interactsh server for OAST testing, exclude OAST based templates
FUZZING:
- -ft, -fuzzing-type string overrides fuzzing type set in template (replace, prefix, postfix, infix)
- -fm, -fuzzing-mode string overrides fuzzing mode set in template (multiple, single)
- -fuzz enable loading fuzzing templates (Deprecated: use -dast instead)
- -dast only run DAST templates
+ -ft, -fuzzing-type string overrides fuzzing type set in template (replace, prefix, postfix, infix)
+ -fm, -fuzzing-mode string overrides fuzzing mode set in template (multiple, single)
+ -fuzz enable loading fuzzing templates (Deprecated: use -dast instead)
+ -dast enable / run dast (fuzz) nuclei templates
+ -dfp, -display-fuzz-points display fuzz points in the output for debugging
+ -fuzz-param-frequency int frequency of uninteresting parameters for fuzzing before skipping (default 10)
+ -fa, -fuzz-aggression string fuzzing aggression level controls payload count for fuzz (low, medium, high) (default "low")
UNCOVER:
-uc, -uncover enable uncover engine
@@ -238,13 +245,15 @@ UNCOVER:
RATE-LIMIT:
-rl, -rate-limit int maximum number of requests to send per second (default 150)
- -rlm, -rate-limit-minute int maximum number of requests to send per minute
+ -rld, -rate-limit-duration value maximum number of requests to send per second (default 1s)
+ -rlm, -rate-limit-minute int maximum number of requests to send per minute (DEPRECATED)
-bs, -bulk-size int maximum number of hosts to be analyzed in parallel per template (default 25)
-c, -concurrency int maximum number of templates to be executed in parallel (default 25)
-hbs, -headless-bulk-size int maximum number of headless hosts to be analyzed in parallel per template (default 10)
-headc, -headless-concurrency int maximum number of headless templates to be executed in parallel (default 10)
-jsc, -js-concurrency int maximum number of javascript runtimes to be executed in parallel (default 120)
-pc, -payload-concurrency int max payload concurrency for each template (default 25)
+ -prc, -probe-concurrency int http probe concurrency with httpx (default 50)
OPTIMIZATIONS:
-timeout int time to wait in seconds before timeout (default 10)
From e13c7cc96703bc6ea687ed3bdaaafab53afcce0a Mon Sep 17 00:00:00 2001
From: Mzack9999
Date: Tue, 11 Jun 2024 12:21:43 +0200
Subject: [PATCH 046/283] testing fastdialer native
---
go.mod | 10 ++---
go.sum | 8 ++++
.../common/hosterrorscache/hosterrorscache.go | 37 +++++++++----------
.../hosterrorscache/hosterrorscache_test.go | 8 +---
4 files changed, 32 insertions(+), 31 deletions(-)
diff --git a/go.mod b/go.mod
index 07c36e7d95..c5e72d5dd8 100644
--- a/go.mod
+++ b/go.mod
@@ -20,12 +20,12 @@ require (
github.com/olekukonko/tablewriter v0.0.5
github.com/pkg/errors v0.9.1
github.com/projectdiscovery/clistats v0.0.20
- github.com/projectdiscovery/fastdialer v0.1.0
- github.com/projectdiscovery/hmap v0.0.45
+ github.com/projectdiscovery/fastdialer v0.1.4-0.20240611102108-26fe40135e03
+ github.com/projectdiscovery/hmap v0.0.46
github.com/projectdiscovery/interactsh v1.1.9
github.com/projectdiscovery/rawhttp v0.1.51
- github.com/projectdiscovery/retryabledns v1.0.60
- github.com/projectdiscovery/retryablehttp-go v1.0.62
+ github.com/projectdiscovery/retryabledns v1.0.63
+ github.com/projectdiscovery/retryablehttp-go v1.0.63
github.com/projectdiscovery/yamldoc-go v1.0.4
github.com/remeh/sizedwaitgroup v1.0.0
github.com/rs/xid v1.5.0
@@ -94,7 +94,7 @@ require (
github.com/projectdiscovery/tlsx v1.1.6
github.com/projectdiscovery/uncover v1.0.8
github.com/projectdiscovery/useragent v0.0.54
- github.com/projectdiscovery/utils v0.1.0
+ github.com/projectdiscovery/utils v0.1.3
github.com/projectdiscovery/wappalyzergo v0.1.1
github.com/redis/go-redis/v9 v9.1.0
github.com/seh-msft/burpxml v1.0.1
diff --git a/go.sum b/go.sum
index ccdec0fcab..394f5fc481 100644
--- a/go.sum
+++ b/go.sum
@@ -834,6 +834,10 @@ github.com/projectdiscovery/dsl v0.1.0 h1:10xoL0V/btU646xH6ioGXKwrGt6grT+BIa6id1
github.com/projectdiscovery/dsl v0.1.0/go.mod h1:JgOjLk54Tl9HG98N2SM5OaKKJ/STpI7yDiKW5rhV+yg=
github.com/projectdiscovery/fastdialer v0.1.0 h1:NiId7eXvYZjiBo83AnIZ6K8ZjzLJU8wGXdRLzuKcRrg=
github.com/projectdiscovery/fastdialer v0.1.0/go.mod h1:jKOWzyHx+Q2sMiYst5HP6tWLQLDFPVgWBR72szn2h8w=
+github.com/projectdiscovery/fastdialer v0.1.4-0.20240611101622-13e3136a3c59 h1:gHx3QQGu7My6SyN9PExd+fZ6XqnBfIIOazTlGz6wwts=
+github.com/projectdiscovery/fastdialer v0.1.4-0.20240611101622-13e3136a3c59/go.mod h1:3UtgUb89y6V3EZ67Ih7IiqSufHZlZBs4t3mE8HpyA54=
+github.com/projectdiscovery/fastdialer v0.1.4-0.20240611102108-26fe40135e03 h1:+wi5E9edG2M6OznZjnMtuGgRwaHx7AckVPu5zNCATE0=
+github.com/projectdiscovery/fastdialer v0.1.4-0.20240611102108-26fe40135e03/go.mod h1:3UtgUb89y6V3EZ67Ih7IiqSufHZlZBs4t3mE8HpyA54=
github.com/projectdiscovery/fasttemplate v0.0.2 h1:h2cISk5xDhlJEinlBQS6RRx0vOlOirB2y3Yu4PJzpiA=
github.com/projectdiscovery/fasttemplate v0.0.2/go.mod h1:XYWWVMxnItd+r0GbjA1GCsUopMw1/XusuQxdyAIHMCw=
github.com/projectdiscovery/freeport v0.0.5 h1:jnd3Oqsl4S8n0KuFkE5Hm8WGDP24ITBvmyw5pFTHS8Q=
@@ -850,6 +854,7 @@ github.com/projectdiscovery/gozero v0.0.2 h1:8fJeaCjxL9tpm33uG/RsCQs6HGM/NE6eA3c
github.com/projectdiscovery/gozero v0.0.2/go.mod h1:d8bZvDWW07LWNYWrwjZ4OO1I0cpkfqaysyDfSs9ibK8=
github.com/projectdiscovery/hmap v0.0.45 h1:D3PinywmpDtIX2lc9bnqaRJoFDPCCxUf4RvHA3Gdr6U=
github.com/projectdiscovery/hmap v0.0.45/go.mod h1:vJ7HYzhDnOjsEc9d43Q2pIWRJ5QvFXdObTTQvaUZaPk=
+github.com/projectdiscovery/hmap v0.0.46/go.mod h1:QleSTt2HrokGNPpymUqb7PQ/2EEHEnaU5qKrrm51Va8=
github.com/projectdiscovery/httpx v1.6.3 h1:TSu3zEKOHGFfMsGeB0lZEFRl6Tq4nPuIgKDXi9NnYpg=
github.com/projectdiscovery/httpx v1.6.3/go.mod h1:9FmsmkT71aTa4WnOyBIPzB8I9c0QAJ9PSVUqs2NkHRQ=
github.com/projectdiscovery/interactsh v1.1.9 h1:b77SaSGrO+DtivmWwqGGY2dmNlQC3Zgmwlaj9L4Oqvc=
@@ -872,8 +877,10 @@ github.com/projectdiscovery/rdap v0.9.1-0.20221108103045-9865884d1917 h1:m03X4gB
github.com/projectdiscovery/rdap v0.9.1-0.20221108103045-9865884d1917/go.mod h1:JxXtZC9e195awe7EynrcnBJmFoad/BNDzW9mzFkK8Sg=
github.com/projectdiscovery/retryabledns v1.0.60 h1:f3TPeLnaJKrl9CbfsTfFTFnJY+UnyArH3mxxUUyR5ZQ=
github.com/projectdiscovery/retryabledns v1.0.60/go.mod h1:T4Su40Wa9lVtRNMfMDFJi00g2T3FbTfwnKKkYON0WgU=
+github.com/projectdiscovery/retryabledns v1.0.63/go.mod h1:lTs48OYJnMFuuBzT+3z3PrZ58K0OUBgP7Y4o3ttBwb0=
github.com/projectdiscovery/retryablehttp-go v1.0.62 h1:ZY09NYRatoELHaPUcdRdWzHIJExjEqceHAfe7u/e9ZY=
github.com/projectdiscovery/retryablehttp-go v1.0.62/go.mod h1:yhRy9Q3z0CuNUHIIPrveaubjVEwF4arKWT/5SQurQEE=
+github.com/projectdiscovery/retryablehttp-go v1.0.63/go.mod h1:ytE1KGi63NFAyLzWqTwZJXwjAg7apgqTnJtaZeZ4qtM=
github.com/projectdiscovery/sarif v0.0.1 h1:C2Tyj0SGOKbCLgHrx83vaE6YkzXEVrMXYRGLkKCr/us=
github.com/projectdiscovery/sarif v0.0.1/go.mod h1:cEYlDu8amcPf6b9dSakcz2nNnJsoz4aR6peERwV+wuQ=
github.com/projectdiscovery/stringsutil v0.0.2 h1:uzmw3IVLJSMW1kEg8eCStG/cGbYYZAja8BH3LqqJXMA=
@@ -886,6 +893,7 @@ github.com/projectdiscovery/useragent v0.0.54 h1:3gM4TnhLSspSM/XkI97YcUazXA3KOkQ
github.com/projectdiscovery/useragent v0.0.54/go.mod h1:dKFkqUb44qx7MpwQDAbhL1t2IrFhcrgI5MlXUZ9E8dM=
github.com/projectdiscovery/utils v0.1.0 h1:r7Z/s2CBktJ0bnSN410lzOhD8S/0IxmzmFxkQudYKps=
github.com/projectdiscovery/utils v0.1.0/go.mod h1:RaBdJLTKF5FKZ/RtMeccqFBtpsSjaggVw6/oPTpDD40=
+github.com/projectdiscovery/utils v0.1.3/go.mod h1:gny8RbNYXE55IoamF6thRDQ8tcJEw+r0FOGAvncz/oQ=
github.com/projectdiscovery/wappalyzergo v0.1.1 h1:HDxuqawatylDiOlfJf4IsabS0wA/Iyvqm7Dn18TVGjU=
github.com/projectdiscovery/wappalyzergo v0.1.1/go.mod h1:wBYGKmA5BQp/NWsAy1q/jSH8N1LHWQ/LV26DuR+KzPM=
github.com/projectdiscovery/yamldoc-go v1.0.4 h1:eZoESapnMw6WAHiVgRwNqvbJEfNHEH148uthhFbG5jE=
diff --git a/pkg/protocols/common/hosterrorscache/hosterrorscache.go b/pkg/protocols/common/hosterrorscache/hosterrorscache.go
index badff95ddd..7eaf147f4b 100644
--- a/pkg/protocols/common/hosterrorscache/hosterrorscache.go
+++ b/pkg/protocols/common/hosterrorscache/hosterrorscache.go
@@ -1,6 +1,7 @@
package hosterrorscache
import (
+ "log"
"net"
"net/url"
"regexp"
@@ -8,7 +9,7 @@ import (
"sync"
"sync/atomic"
- "github.com/bluele/gcache"
+ "github.com/Mzack9999/gcache"
"github.com/projectdiscovery/gologger"
"github.com/projectdiscovery/nuclei/v3/pkg/catalog/config"
"github.com/projectdiscovery/nuclei/v3/pkg/protocols/common/contextargs"
@@ -37,7 +38,7 @@ var (
type Cache struct {
MaxHostError int
verbose bool
- failedTargets gcache.Cache
+ failedTargets gcache.Cache[string, *cacheItem]
TrackError []string
}
@@ -52,7 +53,7 @@ const DefaultMaxHostsCount = 10000
// New returns a new host max errors cache
func New(maxHostError, maxHostsCount int, trackError []string) *Cache {
- gc := gcache.New(maxHostsCount).
+ gc := gcache.New[string, *cacheItem](maxHostsCount).
ARC().
Build()
return &Cache{failedTargets: gc, MaxHostError: maxHostError, TrackError: trackError}
@@ -67,11 +68,7 @@ func (c *Cache) SetVerbose(verbose bool) {
func (c *Cache) Close() {
if config.DefaultConfig.IsDebugArgEnabled(config.DebugArgHostErrorStats) {
items := c.failedTargets.GetALL(false)
- for k, v := range items {
- val, ok := v.(*cacheItem)
- if !ok {
- continue
- }
+ for k, val := range items {
gologger.Info().Label("MaxHostErrorStats").Msgf("Host: %s, Errors: %d", k, val.errors.Load())
}
}
@@ -115,16 +112,15 @@ func (c *Cache) Check(ctx *contextargs.Context) bool {
if err != nil {
return false
}
- existingCacheItemValue := existingCacheItem.(*cacheItem)
- if existingCacheItemValue.isPermanentErr {
+ if existingCacheItem.isPermanentErr {
// skipping permanent errors is expected so verbose instead of info
- gologger.Verbose().Msgf("Skipped %s from target list as found unresponsive permanently: %s", finalValue, existingCacheItemValue.cause)
+ gologger.Verbose().Msgf("Skipped %s from target list as found unresponsive permanently: %s", finalValue, existingCacheItem.cause)
return true
}
- if existingCacheItemValue.errors.Load() >= int32(c.MaxHostError) {
- existingCacheItemValue.Do(func() {
- gologger.Info().Msgf("Skipped %s from target list as found unresponsive %d times", finalValue, existingCacheItemValue.errors.Load())
+ if existingCacheItem.errors.Load() >= int32(c.MaxHostError) {
+ existingCacheItem.Do(func() {
+ gologger.Info().Msgf("Skipped %s from target list as found unresponsive %d times", finalValue, existingCacheItem.errors.Load())
})
return true
}
@@ -133,6 +129,7 @@ func (c *Cache) Check(ctx *contextargs.Context) bool {
// MarkFailed marks a host as failed previously
func (c *Cache) MarkFailed(ctx *contextargs.Context, err error) {
+ log.Printf("%#+v", err)
if !c.checkError(err) {
return
}
@@ -151,9 +148,8 @@ func (c *Cache) MarkFailed(ctx *contextargs.Context, err error) {
_ = c.failedTargets.Set(finalValue, newItem)
return
}
- existingCacheItemValue := existingCacheItem.(*cacheItem)
- existingCacheItemValue.errors.Add(1)
- _ = c.failedTargets.Set(finalValue, existingCacheItemValue)
+ existingCacheItem.errors.Add(1)
+ _ = c.failedTargets.Set(finalValue, existingCacheItem)
}
// GetKeyFromContext returns the key for the cache from the context
@@ -186,10 +182,11 @@ func (c *Cache) checkError(err error) bool {
return false
}
+ // todo: fixed in fastdialer - ref: https://github.com/projectdiscovery/fastdialer/pull/301
// we do not consider timeouts as temporary
- if strings.Contains(errkit.FromError(err).Cause().Error(), "i/o timeout") {
- return true
- }
+ // if strings.Contains(errkit.FromError(err).Cause().Error(), "i/o timeout") {
+ // return true
+ // }
kind := errkit.GetErrorKind(err, nucleierr.ErrTemplateLogic)
switch kind {
diff --git a/pkg/protocols/common/hosterrorscache/hosterrorscache_test.go b/pkg/protocols/common/hosterrorscache/hosterrorscache_test.go
index 6fd04b475b..112690d875 100644
--- a/pkg/protocols/common/hosterrorscache/hosterrorscache_test.go
+++ b/pkg/protocols/common/hosterrorscache/hosterrorscache_test.go
@@ -91,9 +91,7 @@ func TestCacheMarkFailed(t *testing.T) {
require.Nil(t, err)
require.NotNil(t, failedTarget)
- value, ok := failedTarget.(*cacheItem)
- require.True(t, ok)
- require.EqualValues(t, test.expected, value.errors.Load())
+ require.EqualValues(t, test.expected, failedTarget.errors.Load())
}
}
@@ -138,9 +136,7 @@ func TestCacheMarkFailedConcurrent(t *testing.T) {
require.Nil(t, err)
require.NotNil(t, failedTarget)
- value, ok := failedTarget.(*cacheItem)
- require.True(t, ok)
- require.EqualValues(t, test.expected, value.errors.Load())
+ require.EqualValues(t, test.expected, failedTarget.errors.Load())
}
}
From 9f1414e3e8331a5dbdaf0a3cff05e224921f6fb2 Mon Sep 17 00:00:00 2001
From: Mzack9999
Date: Tue, 11 Jun 2024 13:38:28 +0200
Subject: [PATCH 047/283] .
---
go.mod | 8 +++++---
go.sum | 10 ++++------
internal/runner/options.go | 7 +++----
pkg/core/execute_options.go | 2 +-
pkg/installer/versioncheck.go | 2 ++
.../common/hosterrorscache/hosterrorscache.go | 2 --
6 files changed, 15 insertions(+), 16 deletions(-)
diff --git a/go.mod b/go.mod
index 2ff5eb1e4c..139240a98e 100644
--- a/go.mod
+++ b/go.mod
@@ -2,6 +2,8 @@ module github.com/projectdiscovery/nuclei/v3
go 1.21
+replace github.com/projectdiscovery/fastdialer => C:\\Users\\marco\\go\\src\\github.com\\projectdiscovery\\fastdialer
+
require (
github.com/Knetic/govaluate v3.0.1-0.20171022003610-9aa49832a739+incompatible
github.com/andygrunwald/go-jira v1.16.0
@@ -20,12 +22,12 @@ require (
github.com/olekukonko/tablewriter v0.0.5
github.com/pkg/errors v0.9.1
github.com/projectdiscovery/clistats v0.0.20
- github.com/projectdiscovery/fastdialer v0.1.4-0.20240611102108-26fe40135e03
+ github.com/projectdiscovery/fastdialer v0.1.4-0.20240611113719-19b2fb64716e
github.com/projectdiscovery/hmap v0.0.46
github.com/projectdiscovery/interactsh v1.2.0
github.com/projectdiscovery/rawhttp v0.1.51
github.com/projectdiscovery/retryabledns v1.0.63
- github.com/projectdiscovery/retryablehttp-go v1.0.63
+ github.com/projectdiscovery/retryablehttp-go v1.0.64
github.com/projectdiscovery/yamldoc-go v1.0.4
github.com/remeh/sizedwaitgroup v1.0.0
github.com/rs/xid v1.5.0
@@ -94,7 +96,7 @@ require (
github.com/projectdiscovery/tlsx v1.1.6
github.com/projectdiscovery/uncover v1.0.8
github.com/projectdiscovery/useragent v0.0.54
- github.com/projectdiscovery/utils v0.1.3
+ github.com/projectdiscovery/utils v0.1.4-0.20240611113448-0e2f2d33fe1c
github.com/projectdiscovery/wappalyzergo v0.1.4
github.com/redis/go-redis/v9 v9.1.0
github.com/seh-msft/burpxml v1.0.1
diff --git a/go.sum b/go.sum
index c68da12894..8d1b346841 100644
--- a/go.sum
+++ b/go.sum
@@ -832,8 +832,6 @@ github.com/projectdiscovery/clistats v0.0.20 h1:5jO5SLiRJ7f0nDV0ndBNmBeesbROouPo
github.com/projectdiscovery/clistats v0.0.20/go.mod h1:GJ2av0KnOvK0AISQnP8hyDclYIji1LVkx2l0pwnzAu4=
github.com/projectdiscovery/dsl v0.1.2 h1:RdWCEsUqyWsLIQ1k16NX5FiIVbPFvzx9BOlvzK7eeGw=
github.com/projectdiscovery/dsl v0.1.2/go.mod h1:3zwogxGPnPkVUk5OfZQ9Y51sAWdazfQoVZFSPqeEUYA=
-github.com/projectdiscovery/fastdialer v0.1.4-0.20240611102108-26fe40135e03 h1:+wi5E9edG2M6OznZjnMtuGgRwaHx7AckVPu5zNCATE0=
-github.com/projectdiscovery/fastdialer v0.1.4-0.20240611102108-26fe40135e03/go.mod h1:3UtgUb89y6V3EZ67Ih7IiqSufHZlZBs4t3mE8HpyA54=
github.com/projectdiscovery/fasttemplate v0.0.2 h1:h2cISk5xDhlJEinlBQS6RRx0vOlOirB2y3Yu4PJzpiA=
github.com/projectdiscovery/fasttemplate v0.0.2/go.mod h1:XYWWVMxnItd+r0GbjA1GCsUopMw1/XusuQxdyAIHMCw=
github.com/projectdiscovery/freeport v0.0.5 h1:jnd3Oqsl4S8n0KuFkE5Hm8WGDP24ITBvmyw5pFTHS8Q=
@@ -872,8 +870,8 @@ github.com/projectdiscovery/rdap v0.9.1-0.20221108103045-9865884d1917 h1:m03X4gB
github.com/projectdiscovery/rdap v0.9.1-0.20221108103045-9865884d1917/go.mod h1:JxXtZC9e195awe7EynrcnBJmFoad/BNDzW9mzFkK8Sg=
github.com/projectdiscovery/retryabledns v1.0.63 h1:Ijt47ybwf+iIwul606NlPKPN+ouhOYPeA9sLHVgqLG4=
github.com/projectdiscovery/retryabledns v1.0.63/go.mod h1:lTs48OYJnMFuuBzT+3z3PrZ58K0OUBgP7Y4o3ttBwb0=
-github.com/projectdiscovery/retryablehttp-go v1.0.63 h1:y/MpWrAO7702ybOLo/OuyAx6bZ6NJVEf85mGMAU51l8=
-github.com/projectdiscovery/retryablehttp-go v1.0.63/go.mod h1:ytE1KGi63NFAyLzWqTwZJXwjAg7apgqTnJtaZeZ4qtM=
+github.com/projectdiscovery/retryablehttp-go v1.0.64 h1:90VkUuWKm/JW5Tym0u+N1qseKe4gC5Rp048VJBLcxv8=
+github.com/projectdiscovery/retryablehttp-go v1.0.64/go.mod h1:fXVLBcwKwBrCVEgi3FrbLPtSTIczyJEKIlp1WD6VTI0=
github.com/projectdiscovery/sarif v0.0.1 h1:C2Tyj0SGOKbCLgHrx83vaE6YkzXEVrMXYRGLkKCr/us=
github.com/projectdiscovery/sarif v0.0.1/go.mod h1:cEYlDu8amcPf6b9dSakcz2nNnJsoz4aR6peERwV+wuQ=
github.com/projectdiscovery/stringsutil v0.0.2 h1:uzmw3IVLJSMW1kEg8eCStG/cGbYYZAja8BH3LqqJXMA=
@@ -884,8 +882,8 @@ github.com/projectdiscovery/uncover v1.0.8 h1:nE8bYJuwhqk0BEMRQRhNbjPGMy40A9gkiM
github.com/projectdiscovery/uncover v1.0.8/go.mod h1:1SwsNqjwMkJEzJQ7lQr5AHIdYd+BQlyqhO5IwIYmIAM=
github.com/projectdiscovery/useragent v0.0.54 h1:3gM4TnhLSspSM/XkI97YcUazXA3KOkQXEU7TIBly2Fc=
github.com/projectdiscovery/useragent v0.0.54/go.mod h1:dKFkqUb44qx7MpwQDAbhL1t2IrFhcrgI5MlXUZ9E8dM=
-github.com/projectdiscovery/utils v0.1.3 h1:yhHkrbYZA1eOO8e+fPDUvRMS5aUIalyM3Nab7rK4tpg=
-github.com/projectdiscovery/utils v0.1.3/go.mod h1:gny8RbNYXE55IoamF6thRDQ8tcJEw+r0FOGAvncz/oQ=
+github.com/projectdiscovery/utils v0.1.4-0.20240611113448-0e2f2d33fe1c h1:0I/iRtu5nPYle1v8/R33pCLOrH5bziP5Bi0eZURxTQY=
+github.com/projectdiscovery/utils v0.1.4-0.20240611113448-0e2f2d33fe1c/go.mod h1:mXs6OOeG9l/dVchjB2PGvQO3+wuMiE14Y/kmHeKogoM=
github.com/projectdiscovery/wappalyzergo v0.1.4 h1:Q0j5t/c74DlPKRvGYLZFv8npTA7MBW9uwdlsMBn/Yh4=
github.com/projectdiscovery/wappalyzergo v0.1.4/go.mod h1:wBYGKmA5BQp/NWsAy1q/jSH8N1LHWQ/LV26DuR+KzPM=
github.com/projectdiscovery/yamldoc-go v1.0.4 h1:eZoESapnMw6WAHiVgRwNqvbJEfNHEH148uthhFbG5jE=
diff --git a/internal/runner/options.go b/internal/runner/options.go
index 3d36f51123..ebb49fff7e 100644
--- a/internal/runner/options.go
+++ b/internal/runner/options.go
@@ -32,7 +32,6 @@ import (
"github.com/projectdiscovery/nuclei/v3/pkg/utils/yaml"
fileutil "github.com/projectdiscovery/utils/file"
"github.com/projectdiscovery/utils/generic"
- logutil "github.com/projectdiscovery/utils/log"
stringsutil "github.com/projectdiscovery/utils/strings"
)
@@ -322,9 +321,6 @@ func createReportingOptions(options *types.Options) (*reporting.Options, error)
// configureOutput configures the output logging levels to be displayed on the screen
func configureOutput(options *types.Options) {
- // disable standard logger (ref: https://github.com/golang/go/issues/19895)
- defer logutil.DisableDefaultLogger()
-
if options.NoColor {
gologger.DefaultLogger.SetFormatter(formatter.NewCLI(true))
}
@@ -343,6 +339,9 @@ func configureOutput(options *types.Options) {
if options.Silent {
gologger.DefaultLogger.SetMaxLevel(levels.LevelSilent)
}
+
+ // disable standard logger (ref: https://github.com/golang/go/issues/19895)
+ // logutil.DisableDefaultLogger()
}
// loadResolvers loads resolvers from both user-provided flags and file
diff --git a/pkg/core/execute_options.go b/pkg/core/execute_options.go
index 4d27b5f660..a9d97f775b 100644
--- a/pkg/core/execute_options.go
+++ b/pkg/core/execute_options.go
@@ -84,7 +84,7 @@ func (e *Engine) ExecuteScanWithOpts(ctx context.Context, templatesList []*templ
}
// Execute All SelfContained in parallel
- e.executeAllSelfContained(ctx, selfContained, results, selfcontainedWg)
+ // e.executeAllSelfContained(ctx, selfContained, results, selfcontainedWg)
strategyResult := &atomic.Bool{}
switch e.options.ScanStrategy {
diff --git a/pkg/installer/versioncheck.go b/pkg/installer/versioncheck.go
index 7e2e82d6ff..7709ed74f4 100644
--- a/pkg/installer/versioncheck.go
+++ b/pkg/installer/versioncheck.go
@@ -68,6 +68,7 @@ func getpdtmParams(isSDK bool) string {
// UpdateIgnoreFile updates default ignore file by downloading latest ignore file
func UpdateIgnoreFile() error {
+ return nil
resp, err := retryableHttpClient.Get(pdtmNucleiIgnoreFileEndpoint + "?" + getpdtmParams(false))
if err != nil {
return err
@@ -83,6 +84,7 @@ func UpdateIgnoreFile() error {
}
func doVersionCheck(isSDK bool) error {
+ return nil
// we use global retryablehttp client so its not immeditely gc'd if any references are held
// and according our config we have idle connections which are shown as leaked by goleak in tests
// i.e we close all idle connections after our use and it doesn't affect any other part of the code
diff --git a/pkg/protocols/common/hosterrorscache/hosterrorscache.go b/pkg/protocols/common/hosterrorscache/hosterrorscache.go
index 7eaf147f4b..21603394c2 100644
--- a/pkg/protocols/common/hosterrorscache/hosterrorscache.go
+++ b/pkg/protocols/common/hosterrorscache/hosterrorscache.go
@@ -1,7 +1,6 @@
package hosterrorscache
import (
- "log"
"net"
"net/url"
"regexp"
@@ -129,7 +128,6 @@ func (c *Cache) Check(ctx *contextargs.Context) bool {
// MarkFailed marks a host as failed previously
func (c *Cache) MarkFailed(ctx *contextargs.Context, err error) {
- log.Printf("%#+v", err)
if !c.checkError(err) {
return
}
From bc16ae85aeb7fdd998686288ebd1026abe20c32a Mon Sep 17 00:00:00 2001
From: Mzack9999
Date: Tue, 11 Jun 2024 13:39:21 +0200
Subject: [PATCH 048/283] .
---
go.mod | 2 --
go.sum | 2 ++
2 files changed, 2 insertions(+), 2 deletions(-)
diff --git a/go.mod b/go.mod
index 139240a98e..4e6ad4257c 100644
--- a/go.mod
+++ b/go.mod
@@ -2,8 +2,6 @@ module github.com/projectdiscovery/nuclei/v3
go 1.21
-replace github.com/projectdiscovery/fastdialer => C:\\Users\\marco\\go\\src\\github.com\\projectdiscovery\\fastdialer
-
require (
github.com/Knetic/govaluate v3.0.1-0.20171022003610-9aa49832a739+incompatible
github.com/andygrunwald/go-jira v1.16.0
diff --git a/go.sum b/go.sum
index 8d1b346841..2dcf0cd0d7 100644
--- a/go.sum
+++ b/go.sum
@@ -832,6 +832,8 @@ github.com/projectdiscovery/clistats v0.0.20 h1:5jO5SLiRJ7f0nDV0ndBNmBeesbROouPo
github.com/projectdiscovery/clistats v0.0.20/go.mod h1:GJ2av0KnOvK0AISQnP8hyDclYIji1LVkx2l0pwnzAu4=
github.com/projectdiscovery/dsl v0.1.2 h1:RdWCEsUqyWsLIQ1k16NX5FiIVbPFvzx9BOlvzK7eeGw=
github.com/projectdiscovery/dsl v0.1.2/go.mod h1:3zwogxGPnPkVUk5OfZQ9Y51sAWdazfQoVZFSPqeEUYA=
+github.com/projectdiscovery/fastdialer v0.1.4-0.20240611113719-19b2fb64716e h1:wqcy/9pcENeE7bp97gXzq4hfx+lQuWbl68/U28IR70A=
+github.com/projectdiscovery/fastdialer v0.1.4-0.20240611113719-19b2fb64716e/go.mod h1:PcNscfCNrCnrkmvob90yfd+rNDMh6ZOIvRIl6HFPPXE=
github.com/projectdiscovery/fasttemplate v0.0.2 h1:h2cISk5xDhlJEinlBQS6RRx0vOlOirB2y3Yu4PJzpiA=
github.com/projectdiscovery/fasttemplate v0.0.2/go.mod h1:XYWWVMxnItd+r0GbjA1GCsUopMw1/XusuQxdyAIHMCw=
github.com/projectdiscovery/freeport v0.0.5 h1:jnd3Oqsl4S8n0KuFkE5Hm8WGDP24ITBvmyw5pFTHS8Q=
From ac0107c24202a009db8e3b5a7c37db4e786072f3 Mon Sep 17 00:00:00 2001
From: Mzack9999
Date: Tue, 11 Jun 2024 14:58:58 +0200
Subject: [PATCH 049/283] revert
---
internal/runner/options.go | 3 ++-
pkg/core/execute_options.go | 2 +-
pkg/installer/versioncheck.go | 3 +--
3 files changed, 4 insertions(+), 4 deletions(-)
diff --git a/internal/runner/options.go b/internal/runner/options.go
index ebb49fff7e..8798c73113 100644
--- a/internal/runner/options.go
+++ b/internal/runner/options.go
@@ -32,6 +32,7 @@ import (
"github.com/projectdiscovery/nuclei/v3/pkg/utils/yaml"
fileutil "github.com/projectdiscovery/utils/file"
"github.com/projectdiscovery/utils/generic"
+ logutil "github.com/projectdiscovery/utils/log"
stringsutil "github.com/projectdiscovery/utils/strings"
)
@@ -341,7 +342,7 @@ func configureOutput(options *types.Options) {
}
// disable standard logger (ref: https://github.com/golang/go/issues/19895)
- // logutil.DisableDefaultLogger()
+ logutil.DisableDefaultLogger()
}
// loadResolvers loads resolvers from both user-provided flags and file
diff --git a/pkg/core/execute_options.go b/pkg/core/execute_options.go
index a9d97f775b..4d27b5f660 100644
--- a/pkg/core/execute_options.go
+++ b/pkg/core/execute_options.go
@@ -84,7 +84,7 @@ func (e *Engine) ExecuteScanWithOpts(ctx context.Context, templatesList []*templ
}
// Execute All SelfContained in parallel
- // e.executeAllSelfContained(ctx, selfContained, results, selfcontainedWg)
+ e.executeAllSelfContained(ctx, selfContained, results, selfcontainedWg)
strategyResult := &atomic.Bool{}
switch e.options.ScanStrategy {
diff --git a/pkg/installer/versioncheck.go b/pkg/installer/versioncheck.go
index 7709ed74f4..b06f56f1c1 100644
--- a/pkg/installer/versioncheck.go
+++ b/pkg/installer/versioncheck.go
@@ -68,7 +68,6 @@ func getpdtmParams(isSDK bool) string {
// UpdateIgnoreFile updates default ignore file by downloading latest ignore file
func UpdateIgnoreFile() error {
- return nil
resp, err := retryableHttpClient.Get(pdtmNucleiIgnoreFileEndpoint + "?" + getpdtmParams(false))
if err != nil {
return err
@@ -84,11 +83,11 @@ func UpdateIgnoreFile() error {
}
func doVersionCheck(isSDK bool) error {
- return nil
// we use global retryablehttp client so its not immeditely gc'd if any references are held
// and according our config we have idle connections which are shown as leaked by goleak in tests
// i.e we close all idle connections after our use and it doesn't affect any other part of the code
defer retryableHttpClient.HTTPClient.CloseIdleConnections()
+
resp, err := retryableHttpClient.Get(pdtmNucleiVersionEndpoint + "?" + getpdtmParams(isSDK))
if err != nil {
return err
From 61f391ee8a154913ee79f57dc8cf3a73d00f3668 Mon Sep 17 00:00:00 2001
From: mzack
Date: Wed, 12 Jun 2024 00:34:45 +0200
Subject: [PATCH 050/283] .
---
go.mod | 2 +-
go.sum | 2 ++
pkg/protocols/common/hosterrorscache/hosterrorscache.go | 8 +-------
3 files changed, 4 insertions(+), 8 deletions(-)
diff --git a/go.mod b/go.mod
index 4e6ad4257c..53744224d2 100644
--- a/go.mod
+++ b/go.mod
@@ -20,7 +20,7 @@ require (
github.com/olekukonko/tablewriter v0.0.5
github.com/pkg/errors v0.9.1
github.com/projectdiscovery/clistats v0.0.20
- github.com/projectdiscovery/fastdialer v0.1.4-0.20240611113719-19b2fb64716e
+ github.com/projectdiscovery/fastdialer v0.1.4-0.20240611222741-427bffb2ea13
github.com/projectdiscovery/hmap v0.0.46
github.com/projectdiscovery/interactsh v1.2.0
github.com/projectdiscovery/rawhttp v0.1.51
diff --git a/go.sum b/go.sum
index 2dcf0cd0d7..8ed1a24422 100644
--- a/go.sum
+++ b/go.sum
@@ -834,6 +834,8 @@ github.com/projectdiscovery/dsl v0.1.2 h1:RdWCEsUqyWsLIQ1k16NX5FiIVbPFvzx9BOlvzK
github.com/projectdiscovery/dsl v0.1.2/go.mod h1:3zwogxGPnPkVUk5OfZQ9Y51sAWdazfQoVZFSPqeEUYA=
github.com/projectdiscovery/fastdialer v0.1.4-0.20240611113719-19b2fb64716e h1:wqcy/9pcENeE7bp97gXzq4hfx+lQuWbl68/U28IR70A=
github.com/projectdiscovery/fastdialer v0.1.4-0.20240611113719-19b2fb64716e/go.mod h1:PcNscfCNrCnrkmvob90yfd+rNDMh6ZOIvRIl6HFPPXE=
+github.com/projectdiscovery/fastdialer v0.1.4-0.20240611222741-427bffb2ea13 h1:mAehxpgxmvFrKcQ8taA3xQskp3gg6GayAgatTAn37bg=
+github.com/projectdiscovery/fastdialer v0.1.4-0.20240611222741-427bffb2ea13/go.mod h1:PcNscfCNrCnrkmvob90yfd+rNDMh6ZOIvRIl6HFPPXE=
github.com/projectdiscovery/fasttemplate v0.0.2 h1:h2cISk5xDhlJEinlBQS6RRx0vOlOirB2y3Yu4PJzpiA=
github.com/projectdiscovery/fasttemplate v0.0.2/go.mod h1:XYWWVMxnItd+r0GbjA1GCsUopMw1/XusuQxdyAIHMCw=
github.com/projectdiscovery/freeport v0.0.5 h1:jnd3Oqsl4S8n0KuFkE5Hm8WGDP24ITBvmyw5pFTHS8Q=
diff --git a/pkg/protocols/common/hosterrorscache/hosterrorscache.go b/pkg/protocols/common/hosterrorscache/hosterrorscache.go
index 21603394c2..38a7ff3290 100644
--- a/pkg/protocols/common/hosterrorscache/hosterrorscache.go
+++ b/pkg/protocols/common/hosterrorscache/hosterrorscache.go
@@ -180,12 +180,6 @@ func (c *Cache) checkError(err error) bool {
return false
}
- // todo: fixed in fastdialer - ref: https://github.com/projectdiscovery/fastdialer/pull/301
- // we do not consider timeouts as temporary
- // if strings.Contains(errkit.FromError(err).Cause().Error(), "i/o timeout") {
- // return true
- // }
-
kind := errkit.GetErrorKind(err, nucleierr.ErrTemplateLogic)
switch kind {
case nucleierr.ErrTemplateLogic:
@@ -202,7 +196,7 @@ func (c *Cache) checkError(err error) bool {
// these should not be counted as host errors
return false
default:
- // parse error for furthur processing
+ // parse error for further processing
errX := errkit.FromError(err)
tmp := errX.Cause()
cause := tmp.Error()
From 5fd3f0f65d8c02bf97148622bb984d35dab797b8 Mon Sep 17 00:00:00 2001
From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com>
Date: Tue, 11 Jun 2024 23:03:10 +0000
Subject: [PATCH 051/283] chore(deps): bump
github.com/projectdiscovery/ratelimit
Bumps [github.com/projectdiscovery/ratelimit](https://github.com/projectdiscovery/ratelimit) from 0.0.43 to 0.0.45.
- [Release notes](https://github.com/projectdiscovery/ratelimit/releases)
- [Commits](https://github.com/projectdiscovery/ratelimit/compare/v0.0.43...v0.0.45)
---
updated-dependencies:
- dependency-name: github.com/projectdiscovery/ratelimit
dependency-type: direct:production
update-type: version-update:semver-patch
...
Signed-off-by: dependabot[bot]
---
go.mod | 2 +-
go.sum | 6 ++----
2 files changed, 3 insertions(+), 5 deletions(-)
diff --git a/go.mod b/go.mod
index 53744224d2..700d6c1917 100644
--- a/go.mod
+++ b/go.mod
@@ -88,7 +88,7 @@ require (
github.com/projectdiscovery/httpx v1.6.3
github.com/projectdiscovery/mapcidr v1.1.34
github.com/projectdiscovery/n3iwf v0.0.0-20230523120440-b8cd232ff1f5
- github.com/projectdiscovery/ratelimit v0.0.43
+ github.com/projectdiscovery/ratelimit v0.0.45
github.com/projectdiscovery/rdap v0.9.1-0.20221108103045-9865884d1917
github.com/projectdiscovery/sarif v0.0.1
github.com/projectdiscovery/tlsx v1.1.6
diff --git a/go.sum b/go.sum
index 8ed1a24422..57abd0af50 100644
--- a/go.sum
+++ b/go.sum
@@ -832,8 +832,6 @@ github.com/projectdiscovery/clistats v0.0.20 h1:5jO5SLiRJ7f0nDV0ndBNmBeesbROouPo
github.com/projectdiscovery/clistats v0.0.20/go.mod h1:GJ2av0KnOvK0AISQnP8hyDclYIji1LVkx2l0pwnzAu4=
github.com/projectdiscovery/dsl v0.1.2 h1:RdWCEsUqyWsLIQ1k16NX5FiIVbPFvzx9BOlvzK7eeGw=
github.com/projectdiscovery/dsl v0.1.2/go.mod h1:3zwogxGPnPkVUk5OfZQ9Y51sAWdazfQoVZFSPqeEUYA=
-github.com/projectdiscovery/fastdialer v0.1.4-0.20240611113719-19b2fb64716e h1:wqcy/9pcENeE7bp97gXzq4hfx+lQuWbl68/U28IR70A=
-github.com/projectdiscovery/fastdialer v0.1.4-0.20240611113719-19b2fb64716e/go.mod h1:PcNscfCNrCnrkmvob90yfd+rNDMh6ZOIvRIl6HFPPXE=
github.com/projectdiscovery/fastdialer v0.1.4-0.20240611222741-427bffb2ea13 h1:mAehxpgxmvFrKcQ8taA3xQskp3gg6GayAgatTAn37bg=
github.com/projectdiscovery/fastdialer v0.1.4-0.20240611222741-427bffb2ea13/go.mod h1:PcNscfCNrCnrkmvob90yfd+rNDMh6ZOIvRIl6HFPPXE=
github.com/projectdiscovery/fasttemplate v0.0.2 h1:h2cISk5xDhlJEinlBQS6RRx0vOlOirB2y3Yu4PJzpiA=
@@ -866,8 +864,8 @@ github.com/projectdiscovery/n3iwf v0.0.0-20230523120440-b8cd232ff1f5 h1:L/e8z8yw
github.com/projectdiscovery/n3iwf v0.0.0-20230523120440-b8cd232ff1f5/go.mod h1:pGW2ncnTxTxHtP9wzcIJAB+3/NMp6IiuQWd2NK7K+oc=
github.com/projectdiscovery/networkpolicy v0.0.8 h1:XvfBaBwSDNTesSfNQP9VLk3HX9I7x7gHm028TJ5XwI8=
github.com/projectdiscovery/networkpolicy v0.0.8/go.mod h1:xnjNqhemxUPxU+UD5Jgsc3+K8IVmcqT1SJeo6UzMtkI=
-github.com/projectdiscovery/ratelimit v0.0.43 h1:9yYCnF+NW/vIRVmyvmyCsIph9Z3nsCDka2BrRciDg4c=
-github.com/projectdiscovery/ratelimit v0.0.43/go.mod h1:pWKRf/OY4uKtjelBLS5G3mclgvlwpBtbjcYpxdLQsL0=
+github.com/projectdiscovery/ratelimit v0.0.45 h1:h28oF+hJ0CHcdBZozT1Go7ppWmzTxSXDKNNh2G1Ot9Q=
+github.com/projectdiscovery/ratelimit v0.0.45/go.mod h1:1vSJUseDS7SjNwIBi9wNRcgsMKNTLxy/GfdlLFVbgI4=
github.com/projectdiscovery/rawhttp v0.1.51 h1:T4sskU2fsX9zNR7nyae0tD/0fliIs73KwBzyoJwBs10=
github.com/projectdiscovery/rawhttp v0.1.51/go.mod h1:ejBaw3dCfJSfbtFv0ib6YfFSjQzt7ZdRVpQWJASxIUs=
github.com/projectdiscovery/rdap v0.9.1-0.20221108103045-9865884d1917 h1:m03X4gBVSorSzvmm0bFa7gDV4QNSOWPL/fgZ4kTXBxk=
From 27eedc0ac68a485045c81b06aa9140adbc7f0735 Mon Sep 17 00:00:00 2001
From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com>
Date: Tue, 11 Jun 2024 23:37:17 +0000
Subject: [PATCH 052/283] chore(deps): bump
github.com/Azure/azure-sdk-for-go/sdk/azidentity
Bumps [github.com/Azure/azure-sdk-for-go/sdk/azidentity](https://github.com/Azure/azure-sdk-for-go) from 1.3.0 to 1.6.0.
- [Release notes](https://github.com/Azure/azure-sdk-for-go/releases)
- [Changelog](https://github.com/Azure/azure-sdk-for-go/blob/main/documentation/release.md)
- [Commits](https://github.com/Azure/azure-sdk-for-go/compare/sdk/azcore/v1.3.0...sdk/azcore/v1.6.0)
---
updated-dependencies:
- dependency-name: github.com/Azure/azure-sdk-for-go/sdk/azidentity
dependency-type: direct:production
...
Signed-off-by: dependabot[bot]
---
go.mod | 25 +++++++++++++------------
go.sum | 48 +++++++++++++++++++++++++++---------------------
2 files changed, 40 insertions(+), 33 deletions(-)
diff --git a/go.mod b/go.mod
index 700d6c1917..6970a332a2 100644
--- a/go.mod
+++ b/go.mod
@@ -38,15 +38,15 @@ require (
github.com/weppos/publicsuffix-go v0.30.2
github.com/xanzy/go-gitlab v0.84.0
go.uber.org/multierr v1.11.0
- golang.org/x/net v0.25.0
+ golang.org/x/net v0.26.0
golang.org/x/oauth2 v0.18.0
- golang.org/x/text v0.15.0
+ golang.org/x/text v0.16.0
gopkg.in/yaml.v2 v2.4.0
)
require (
code.gitea.io/sdk/gitea v0.17.0
- github.com/Azure/azure-sdk-for-go/sdk/azidentity v1.3.0
+ github.com/Azure/azure-sdk-for-go/sdk/azidentity v1.6.0
github.com/Azure/azure-sdk-for-go/sdk/storage/azblob v1.1.0
github.com/DataDog/gostackparse v0.6.0
github.com/Masterminds/semver/v3 v3.2.1
@@ -101,7 +101,7 @@ require (
github.com/stretchr/testify v1.9.0
github.com/tarunKoyalwar/goleak v0.0.0-20240429141123-0efa90dbdcf9
github.com/zmap/zgrab2 v0.1.8-0.20230806160807-97ba87c0e706
- golang.org/x/term v0.20.0
+ golang.org/x/term v0.21.0
gopkg.in/yaml.v3 v3.0.1
moul.io/http2curl v1.0.0
)
@@ -109,11 +109,11 @@ require (
require (
aead.dev/minisign v0.2.0 // indirect
dario.cat/mergo v1.0.0 // indirect
- github.com/Azure/azure-sdk-for-go/sdk/azcore v1.7.0 // indirect
- github.com/Azure/azure-sdk-for-go/sdk/internal v1.3.0 // indirect
+ github.com/Azure/azure-sdk-for-go/sdk/azcore v1.11.1 // indirect
+ github.com/Azure/azure-sdk-for-go/sdk/internal v1.8.0 // indirect
github.com/Azure/go-ansiterm v0.0.0-20230124172434-306776ec8161 // indirect
github.com/Azure/go-ntlmssp v0.0.0-20221128193559-754e69321358 // indirect
- github.com/AzureAD/microsoft-authentication-library-for-go v1.0.0 // indirect
+ github.com/AzureAD/microsoft-authentication-library-for-go v1.2.2 // indirect
github.com/Nvveen/Gotty v0.0.0-20120604004816-cd527374f1e5 // indirect
github.com/VividCortex/ewma v1.2.0 // indirect
github.com/andybalholm/brotli v1.1.0 // indirect
@@ -156,6 +156,7 @@ require (
github.com/go-sourcemap/sourcemap v2.1.4+incompatible // indirect
github.com/goccy/go-json v0.10.2 // indirect
github.com/gogo/protobuf v1.3.2 // indirect
+ github.com/golang-jwt/jwt/v5 v5.2.1 // indirect
github.com/golang-sql/civil v0.0.0-20190719163853-cb61b32ac6fe // indirect
github.com/golang-sql/sqlexp v0.1.0 // indirect
github.com/google/certificate-transparency-go v1.1.4 // indirect
@@ -197,7 +198,7 @@ require (
github.com/perimeterx/marshmallow v1.1.5 // indirect
github.com/pierrec/lz4/v4 v4.1.21 // indirect
github.com/pjbgf/sha1cd v0.3.0 // indirect
- github.com/pkg/browser v0.0.0-20210911075715-681adbf594b8 // indirect
+ github.com/pkg/browser v0.0.0-20240102092130-5ac0b6a4141c // indirect
github.com/projectdiscovery/asnmap v1.1.0 // indirect
github.com/projectdiscovery/cdncheck v1.0.9 // indirect
github.com/projectdiscovery/freeport v0.0.5 // indirect
@@ -263,7 +264,7 @@ require (
github.com/golang/protobuf v1.5.4 // indirect
github.com/golang/snappy v0.0.4 // indirect
github.com/google/go-querystring v1.1.0 // indirect
- github.com/google/uuid v1.3.1 // indirect
+ github.com/google/uuid v1.6.0 // indirect
github.com/gorilla/css v1.0.1 // indirect
github.com/hashicorp/go-cleanhttp v0.5.2 // indirect
github.com/hashicorp/go-retryablehttp v0.7.2 // indirect
@@ -301,12 +302,12 @@ require (
go.etcd.io/bbolt v1.3.10 // indirect
go.uber.org/zap v1.25.0 // indirect
goftp.io/server/v2 v2.0.1 // indirect
- golang.org/x/crypto v0.23.0 // indirect
+ golang.org/x/crypto v0.24.0 // indirect
golang.org/x/exp v0.0.0-20240506185415-9bf2ced13842
golang.org/x/mod v0.17.0 // indirect
- golang.org/x/sys v0.20.0 // indirect
+ golang.org/x/sys v0.21.0 // indirect
golang.org/x/time v0.5.0 // indirect
- golang.org/x/tools v0.21.0
+ golang.org/x/tools v0.21.1-0.20240508182429-e35e4ccd0d2d
google.golang.org/appengine v1.6.8 // indirect
google.golang.org/protobuf v1.33.0 // indirect
gopkg.in/alecthomas/kingpin.v2 v2.2.6 // indirect
diff --git a/go.sum b/go.sum
index 57abd0af50..a3702b0fdb 100644
--- a/go.sum
+++ b/go.sum
@@ -41,14 +41,14 @@ dmitri.shuralyov.com/gpu/mtl v0.0.0-20190408044501-666a987793e9/go.mod h1:H6x//7
git.mills.io/prologic/smtpd v0.0.0-20210710122116-a525b76c287a h1:3i+FJ7IpSZHL+VAjtpQeZCRhrpP0odl5XfoLBY4fxJ8=
git.mills.io/prologic/smtpd v0.0.0-20210710122116-a525b76c287a/go.mod h1:C7hXLmFmPYPjIDGfQl1clsmQ5TMEQfmzWTrJk475bUs=
github.com/Azure/azure-sdk-for-go/sdk/azcore v0.19.0/go.mod h1:h6H6c8enJmmocHUbLiiGY6sx7f9i+X3m1CHdd5c6Rdw=
-github.com/Azure/azure-sdk-for-go/sdk/azcore v1.7.0 h1:8q4SaHjFsClSvuVne0ID/5Ka8u3fcIHyqkLjcFpNRHQ=
-github.com/Azure/azure-sdk-for-go/sdk/azcore v1.7.0/go.mod h1:bjGvMhVMb+EEm3VRNQawDMUyMMjo+S5ewNjflkep/0Q=
+github.com/Azure/azure-sdk-for-go/sdk/azcore v1.11.1 h1:E+OJmp2tPvt1W+amx48v1eqbjDYsgN+RzP4q16yV5eM=
+github.com/Azure/azure-sdk-for-go/sdk/azcore v1.11.1/go.mod h1:a6xsAQUZg+VsS3TJ05SRp524Hs4pZ/AeFSr5ENf0Yjo=
github.com/Azure/azure-sdk-for-go/sdk/azidentity v0.11.0/go.mod h1:HcM1YX14R7CJcghJGOYCgdezslRSVzqwLf/q+4Y2r/0=
-github.com/Azure/azure-sdk-for-go/sdk/azidentity v1.3.0 h1:vcYCAze6p19qBW7MhZybIsqD8sMV8js0NyQM8JDnVtg=
-github.com/Azure/azure-sdk-for-go/sdk/azidentity v1.3.0/go.mod h1:OQeznEEkTZ9OrhHJoDD8ZDq51FHgXjqtP9z6bEwBq9U=
+github.com/Azure/azure-sdk-for-go/sdk/azidentity v1.6.0 h1:U2rTu3Ef+7w9FHKIAXM6ZyqF3UOWJZ12zIm8zECAFfg=
+github.com/Azure/azure-sdk-for-go/sdk/azidentity v1.6.0/go.mod h1:9kIvujWAA58nmPmWB1m23fyWic1kYZMxD9CxaWn4Qpg=
github.com/Azure/azure-sdk-for-go/sdk/internal v0.7.0/go.mod h1:yqy467j36fJxcRV2TzfVZ1pCb5vxm4BtZPUdYWe/Xo8=
-github.com/Azure/azure-sdk-for-go/sdk/internal v1.3.0 h1:sXr+ck84g/ZlZUOZiNELInmMgOsuGwdjjVkEIde0OtY=
-github.com/Azure/azure-sdk-for-go/sdk/internal v1.3.0/go.mod h1:okt5dMMTOFjX/aovMlrjvvXoPMBVSPzk9185BT0+eZM=
+github.com/Azure/azure-sdk-for-go/sdk/internal v1.8.0 h1:jBQA3cKT4L2rWMpgE7Yt3Hwh2aUj8KXjIGLxjHeYNNo=
+github.com/Azure/azure-sdk-for-go/sdk/internal v1.8.0/go.mod h1:4OG6tQ9EOP/MT0NMjDlRzWoVFxfu9rN9B2X+tlSVktg=
github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/storage/armstorage v1.2.0 h1:Ma67P/GGprNwsslzEH6+Kb8nybI8jpDTm4Wmzu2ReK8=
github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/storage/armstorage v1.2.0/go.mod h1:c+Lifp3EDEamAkPVzMooRNOK6CZjNSdEnf1A7jsI9u4=
github.com/Azure/azure-sdk-for-go/sdk/storage/azblob v1.1.0 h1:nVocQV40OQne5613EeLayJiRAJuKlBGy+m22qWG+WRg=
@@ -57,8 +57,8 @@ github.com/Azure/go-ansiterm v0.0.0-20230124172434-306776ec8161 h1:L/gRVlceqvL25
github.com/Azure/go-ansiterm v0.0.0-20230124172434-306776ec8161/go.mod h1:xomTg63KZ2rFqZQzSB4Vz2SUXa1BpHTVz9L5PTmPC4E=
github.com/Azure/go-ntlmssp v0.0.0-20221128193559-754e69321358 h1:mFRzDkZVAjdal+s7s0MwaRv9igoPqLRdzOLzw/8Xvq8=
github.com/Azure/go-ntlmssp v0.0.0-20221128193559-754e69321358/go.mod h1:chxPXzSsl7ZWRAuOIE23GDNzjWuZquvFlgA8xmpunjU=
-github.com/AzureAD/microsoft-authentication-library-for-go v1.0.0 h1:OBhqkivkhkMqLPymWEppkm7vgPQY2XsHoEkaMQ0AdZY=
-github.com/AzureAD/microsoft-authentication-library-for-go v1.0.0/go.mod h1:kgDmCTgBzIEPFElEF+FK0SdjAor06dRq2Go927dnQ6o=
+github.com/AzureAD/microsoft-authentication-library-for-go v1.2.2 h1:XHOnouVk1mxXfQidrMEnLlPk9UMeRtyBTnEFtxkV0kU=
+github.com/AzureAD/microsoft-authentication-library-for-go v1.2.2/go.mod h1:wP83P5OoQ5p6ip3ScPr0BAq0BvuPAvacpEuSzyouqAI=
github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU=
github.com/BurntSushi/xgb v0.0.0-20160522181843-27f122750802/go.mod h1:IVnqGOEym/WlBOVXweHU+Q+/VP0lqqI8lqeDx9IjBqo=
github.com/DataDog/gostackparse v0.6.0 h1:egCGQviIabPwsyoWpGvIBGrEnNWez35aEO7OJ1vBI4o=
@@ -273,7 +273,6 @@ github.com/dlclark/regexp2 v1.4.1-0.20201116162257-a2a8dda75c91/go.mod h1:2pZnwu
github.com/dlclark/regexp2 v1.7.0/go.mod h1:DHkYz0B9wPfa6wondMfaivmHpzrQ3v9q8cnmRbL6yW8=
github.com/dlclark/regexp2 v1.11.0 h1:G/nrcoOa7ZXlpoa/91N3X7mM3r8eIlMBBJZvsz/mxKI=
github.com/dlclark/regexp2 v1.11.0/go.mod h1:DHkYz0B9wPfa6wondMfaivmHpzrQ3v9q8cnmRbL6yW8=
-github.com/dnaeon/go-vcr v1.2.0 h1:zHCHvJYTMh1N7xnV7zf1m1GPBF9Ad0Jk/whtQ1663qI=
github.com/dnaeon/go-vcr v1.2.0/go.mod h1:R4UdLID7HZT3taECzJs4YgbbH6PIGXB6W/sc5OLb6RQ=
github.com/docker/cli v24.0.5+incompatible h1:WeBimjvS0eKdH4Ygx+ihVq1Q++xg36M/rMi4aXAvodc=
github.com/docker/cli v24.0.5+incompatible/go.mod h1:JLrzqnKDaYBop7H2jaqPtU4hHvMKP+vjCwu2uszcLI8=
@@ -416,6 +415,8 @@ github.com/golang-jwt/jwt v3.2.2+incompatible/go.mod h1:8pz2t5EyA70fFQQSrl6XZXzq
github.com/golang-jwt/jwt/v4 v4.4.2/go.mod h1:m21LjoU+eqJr34lmDMbreY2eSTRJ1cv77w39/MY0Ch0=
github.com/golang-jwt/jwt/v4 v4.5.0 h1:7cYmW1XlMY7h7ii7UhUyChSgS5wUJEnm9uZVTGqOWzg=
github.com/golang-jwt/jwt/v4 v4.5.0/go.mod h1:m21LjoU+eqJr34lmDMbreY2eSTRJ1cv77w39/MY0Ch0=
+github.com/golang-jwt/jwt/v5 v5.2.1 h1:OuVbFODueb089Lh128TAcimifWaLhJwVflnrgM17wHk=
+github.com/golang-jwt/jwt/v5 v5.2.1/go.mod h1:pqrtFR0X4osieyHYxtmOUWsAWrfe1Q5UVIyoH402zdk=
github.com/golang-sql/civil v0.0.0-20190719163853-cb61b32ac6fe h1:lXe2qZdvpiX5WZkZR4hgp4KJVfY3nMkvmwbVkpv1rVY=
github.com/golang-sql/civil v0.0.0-20190719163853-cb61b32ac6fe/go.mod h1:8vg3r2VgvsThLBIFL93Qb5yWzgyZWhEmBwUJWevAkK0=
github.com/golang-sql/sqlexp v0.1.0 h1:ZCD6MBpcuOVfGVqsEmY5/4FtYiKz6tSyUv9LPEDei6A=
@@ -501,8 +502,8 @@ github.com/google/renameio v0.1.0/go.mod h1:KWCgfxg9yswjAJkECMjeO8J8rahYeXnNhOm4
github.com/google/shlex v0.0.0-20191202100458-e7afc7fbc510 h1:El6M4kTTCOh6aBiKaUGG7oYTSPP8MxqL4YI3kZKwcP4=
github.com/google/shlex v0.0.0-20191202100458-e7afc7fbc510/go.mod h1:pupxD2MaaD3pAXIBCelhxNneeOaAeabZDe5s4K6zSpQ=
github.com/google/uuid v1.0.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
-github.com/google/uuid v1.3.1 h1:KjJaJ9iWZ3jOFZIf1Lqf4laDRCasjl0BCmnEGxkdLb4=
-github.com/google/uuid v1.3.1/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
+github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0=
+github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
github.com/googleapis/gax-go/v2 v2.0.4/go.mod h1:0Wqv26UfaUD9n4G6kQubkQ+KchISgw+vpHVxEJEs9eg=
github.com/googleapis/gax-go/v2 v2.0.5/go.mod h1:DWXyrwAJ9X0FpwwEdw+IPEYBICEFu5mhpdKc/us6bOk=
github.com/gopherjs/gopherjs v0.0.0-20181017120253-0766667cb4d1 h1:EGx4pi6eqNxGaHF6qqu48+N2wcFQ5qg5FXgOdqsJ5d8=
@@ -808,8 +809,8 @@ github.com/pierrec/lz4/v4 v4.1.21/go.mod h1:gZWDp/Ze/IJXGXf23ltt2EXimqmTUXEy0GFu
github.com/pjbgf/sha1cd v0.3.0 h1:4D5XXmUUBUl/xQ6IjCkEAbqXskkq/4O7LmGn0AqMDs4=
github.com/pjbgf/sha1cd v0.3.0/go.mod h1:nZ1rrWOcGJ5uZgEEVL1VUM9iRQiZvWdbZjkKyFzPPsI=
github.com/pkg/browser v0.0.0-20180916011732-0a3d74bf9ce4/go.mod h1:4OwLy04Bl9Ef3GJJCoec+30X3LQs/0/m4HFRt/2LUSA=
-github.com/pkg/browser v0.0.0-20210911075715-681adbf594b8 h1:KoWmjvw+nsYOo29YJK9vDA65RGE3NrOnUtO7a+RF9HU=
-github.com/pkg/browser v0.0.0-20210911075715-681adbf594b8/go.mod h1:HKlIX3XHQyzLZPlr7++PzdhaXEj94dEiJgZDTsxEqUI=
+github.com/pkg/browser v0.0.0-20240102092130-5ac0b6a4141c h1:+mdjkGKdHQG3305AYmdv1U2eRNDiU2ErMBj1gwrq8eQ=
+github.com/pkg/browser v0.0.0-20240102092130-5ac0b6a4141c/go.mod h1:7rwL4CYBLnjLxUqIJNnCWiEdr3bn6IUYi15bNlnbCCU=
github.com/pkg/errors v0.8.0/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4=
@@ -1183,8 +1184,9 @@ golang.org/x/crypto v0.6.0/go.mod h1:OFC/31mSvZgRz0V1QTNCzfAI1aIRzbiufJtkMIlEp58
golang.org/x/crypto v0.7.0/go.mod h1:pYwdfH91IfpZVANVyUOhSIPZaFoJGxTFbZhFTx+dXZU=
golang.org/x/crypto v0.14.0/go.mod h1:MVFd36DqK4CsrnJYDkBA3VC4m2GkXAM0PvzMCn4JQf4=
golang.org/x/crypto v0.19.0/go.mod h1:Iy9bg/ha4yyC70EfRS8jz+B6ybOBKMaSxLj6P6oBDfU=
-golang.org/x/crypto v0.23.0 h1:dIJU/v2J8Mdglj/8rJ6UUOM3Zc9zLZxVZwwxMooUSAI=
golang.org/x/crypto v0.23.0/go.mod h1:CKFgDieR+mRhux2Lsu27y0fO304Db0wZe70UKqHu0v8=
+golang.org/x/crypto v0.24.0 h1:mnl8DM0o513X8fdIkmyFE/5hTYxbwYOjDS/+rK6qpRI=
+golang.org/x/crypto v0.24.0/go.mod h1:Z1PMYSOR5nyMcyAVAIQSKCDwalqy85Aqn1x3Ws4L5DM=
golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA=
golang.org/x/exp v0.0.0-20190306152737-a1d7652674e8/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA=
golang.org/x/exp v0.0.0-20190510132918-efd6b22b2522/go.mod h1:ZjyILWgesfNpC6sMxTJOJm9Kp84zZh5NQWvqDGG3Qr8=
@@ -1278,8 +1280,9 @@ golang.org/x/net v0.9.0/go.mod h1:d48xBJpPfHeWQsugry2m+kC02ZBRGRgulfHnEXEuWns=
golang.org/x/net v0.10.0/go.mod h1:0qNGK6F8kojg2nk9dLZ2mShWaEBan6FAoqfSigmmuDg=
golang.org/x/net v0.17.0/go.mod h1:NxSsAGuq816PNPmqtQdLE42eU2Fs7NoRIZrHJAlaCOE=
golang.org/x/net v0.21.0/go.mod h1:bIjVDfnllIU7BJ2DNgfnXvpSvtn8VRwhlsaeUTyUS44=
-golang.org/x/net v0.25.0 h1:d/OCCoBEUq33pjydKrGQhw7IlUPI2Oylr+8qLx49kac=
golang.org/x/net v0.25.0/go.mod h1:JkAGAh7GEvH74S6FOH42FLoXpXbE/aqXSrIQjXgsiwM=
+golang.org/x/net v0.26.0 h1:soB7SVo0PWrY4vPW/+ay0jKDNScG2X9wFeYlXIvJsOQ=
+golang.org/x/net v0.26.0/go.mod h1:5YKkiSynbBIh3p6iOc/vibscux0x38BZDkn8sCUPxHE=
golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U=
golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw=
golang.org/x/oauth2 v0.0.0-20190604053449-0f29369cfe45/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw=
@@ -1356,7 +1359,6 @@ golang.org/x/sys v0.0.0-20210228012217-479acdf4ea46/go.mod h1:h1NjWce9XRLGQEsW7w
golang.org/x/sys v0.0.0-20210423082822-04245dca01da/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20210603081109-ebe580a85c40/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
-golang.org/x/sys v0.0.0-20210616045830-e2b7044e8c71/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.0.0-20210616094352-59db8d763f22/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.0.0-20210630005230-0f9fa26af87c/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.0.0-20210927094055-39ccf1dd6fa6/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
@@ -1371,6 +1373,7 @@ golang.org/x/sys v0.0.0-20220715151400-c0bba94af5f8/go.mod h1:oPkhp1MJrh7nUepCBc
golang.org/x/sys v0.0.0-20220722155257-8c9f86f7a55f/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.0.0-20220728004956-3c1f35247d10/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.0.0-20220811171246-fbc7d0a398ab/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
+golang.org/x/sys v0.1.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.3.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.4.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.5.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
@@ -1380,8 +1383,9 @@ golang.org/x/sys v0.8.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.11.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.13.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.17.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
-golang.org/x/sys v0.20.0 h1:Od9JTbYCk261bKm4M/mw7AklTlFYIa0bIp9BgSm1S8Y=
golang.org/x/sys v0.20.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
+golang.org/x/sys v0.21.0 h1:rF+pYz3DAGSQAxAu1CbC7catZg4ebC4UIeIhKxBZvws=
+golang.org/x/sys v0.21.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
golang.org/x/term v0.0.0-20201117132131-f5c789dd3221/go.mod h1:Nr5EML6q2oocZ2LXRh80K7BxOlk5/8JxuGnuhpl+muw=
golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo=
golang.org/x/term v0.0.0-20210220032956-6a3ed077a48d/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo=
@@ -1393,8 +1397,9 @@ golang.org/x/term v0.7.0/go.mod h1:P32HKFT3hSsZrRxla30E9HqToFYAQPCMs/zFMBUFqPY=
golang.org/x/term v0.8.0/go.mod h1:xPskH00ivmX89bAKVGSKKtLOWNx2+17Eiy94tnKShWo=
golang.org/x/term v0.13.0/go.mod h1:LTmsnFJwVN6bCy1rVCoS+qHT1HhALEFxKncY3WNNh4U=
golang.org/x/term v0.17.0/go.mod h1:lLRBjIVuehSbZlaOtGMbcMncT+aqLLLmKrsjNrUguwk=
-golang.org/x/term v0.20.0 h1:VnkxpohqXaOBYJtBmEppKUG6mXpi+4O6purfc2+sMhw=
golang.org/x/term v0.20.0/go.mod h1:8UkIAJTvZgivsXaD6/pH6U9ecQzZ45awqEOzuCvwpFY=
+golang.org/x/term v0.21.0 h1:WVXCp+/EBEHOj53Rvu+7KiT/iElMrO8ACK16SMZ3jaA=
+golang.org/x/term v0.21.0/go.mod h1:ooXLefLobQVslOqselCNF4SxFAaoS6KujMbsGzSDmX0=
golang.org/x/text v0.0.0-20170915032832-14c0d48ead0c/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
golang.org/x/text v0.3.1-0.20180807135948-17ff2d5776d2/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
@@ -1410,8 +1415,9 @@ golang.org/x/text v0.8.0/go.mod h1:e1OnstbJyHTd6l/uOt8jFFHp6TRDWZR/bV3emEE/zU8=
golang.org/x/text v0.9.0/go.mod h1:e1OnstbJyHTd6l/uOt8jFFHp6TRDWZR/bV3emEE/zU8=
golang.org/x/text v0.13.0/go.mod h1:TvPlkZtksWOMsz7fbANvkp4WM8x/WCo/om8BMLbz+aE=
golang.org/x/text v0.14.0/go.mod h1:18ZOQIKpY8NJVqYksKHtTdi31H5itFRjB5/qKTNYzSU=
-golang.org/x/text v0.15.0 h1:h1V/4gjBv8v9cjcR6+AR5+/cIYK5N/WAgiv4xlsEtAk=
golang.org/x/text v0.15.0/go.mod h1:18ZOQIKpY8NJVqYksKHtTdi31H5itFRjB5/qKTNYzSU=
+golang.org/x/text v0.16.0 h1:a94ExnEXNtEwYLGJSIUxnWoxoRz/ZcCsV63ROupILh4=
+golang.org/x/text v0.16.0/go.mod h1:GhwF1Be+LQoKShO3cGOHzqOgRrGaYc9AvblQOmPVHnI=
golang.org/x/time v0.0.0-20180412165947-fbb02b2291d2/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
golang.org/x/time v0.0.0-20181108054448-85acf8d2951c/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
golang.org/x/time v0.0.0-20190308202827-9d24e82272b4/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
@@ -1469,8 +1475,8 @@ golang.org/x/tools v0.0.0-20200825202427-b303f430e36d/go.mod h1:njjCfa9FT2d7l9Bc
golang.org/x/tools v0.0.0-20210106214847-113979e3529a/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA=
golang.org/x/tools v0.1.12/go.mod h1:hNGJHUnrk76NpqgfD5Aqm5Crs+Hm0VOH/i9J2+nxYbc=
golang.org/x/tools v0.6.0/go.mod h1:Xwgl3UAJ/d3gWutnCtw505GrjyAbvKui8lOU390QaIU=
-golang.org/x/tools v0.21.0 h1:qc0xYgIbsSDt9EyWz05J5wfa7LOVW0YTLOXrqdLAWIw=
-golang.org/x/tools v0.21.0/go.mod h1:aiJjzUbINMkxbQROHiO6hDPo2LHcIPhhQsa9DLh0yGk=
+golang.org/x/tools v0.21.1-0.20240508182429-e35e4ccd0d2d h1:vU5i/LfpvrRCpgM/VPfJLg5KjxD3E+hfT1SH+d9zLwg=
+golang.org/x/tools v0.21.1-0.20240508182429-e35e4ccd0d2d/go.mod h1:aiJjzUbINMkxbQROHiO6hDPo2LHcIPhhQsa9DLh0yGk=
golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
From 1e85e70d990cb463beb0e860fe4511c2605f90c1 Mon Sep 17 00:00:00 2001
From: Mzack9999
Date: Thu, 13 Jun 2024 17:24:07 +0200
Subject: [PATCH 053/283] removed as recommended
---
lib/multi.go | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/lib/multi.go b/lib/multi.go
index a441c73fbe..44c13ddfe6 100644
--- a/lib/multi.go
+++ b/lib/multi.go
@@ -128,7 +128,7 @@ func (e *ThreadSafeNucleiEngine) ExecuteNucleiWithOptsCtx(ctx context.Context, t
return err
}
}
- defer tmpEngine.closeInternal()
+
// create ephemeral nuclei objects/instances/types using base nuclei engine
unsafeOpts, err := createEphemeralObjects(ctx, e.eng, tmpEngine.opts)
if err != nil {
From baf7e366585f070aef14f73853685e1aae377f2f Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?=C3=93scar=20Mar=C3=ADn?=
Date: Fri, 14 Jun 2024 20:24:52 +0200
Subject: [PATCH 054/283] issue 5212 timeout added to trace and error logs
using ts switch (#5292)
---
pkg/output/output.go | 19 ++++++++++++-------
1 file changed, 12 insertions(+), 7 deletions(-)
diff --git a/pkg/output/output.go b/pkg/output/output.go
index 449b73a2d1..8044c0aed9 100644
--- a/pkg/output/output.go
+++ b/pkg/output/output.go
@@ -304,13 +304,14 @@ func (w *StandardWriter) Write(event *ResultEvent) error {
// JSONLogRequest is a trace/error log request written to file
type JSONLogRequest struct {
- Template string `json:"template"`
- Type string `json:"type"`
- Input string `json:"input"`
- Address string `json:"address"`
- Error string `json:"error"`
- Kind string `json:"kind,omitempty"`
- Attrs interface{} `json:"attrs,omitempty"`
+ Template string `json:"template"`
+ Type string `json:"type"`
+ Input string `json:"input"`
+ Timestamp *time.Time `json:"timestamp,omitempty"`
+ Address string `json:"address"`
+ Error string `json:"error"`
+ Kind string `json:"kind,omitempty"`
+ Attrs interface{} `json:"attrs,omitempty"`
}
// Request writes a log the requests trace log
@@ -323,6 +324,10 @@ func (w *StandardWriter) Request(templatePath, input, requestType string, reques
Input: input,
Type: requestType,
}
+ if w.timestamp {
+ ts := time.Now()
+ request.Timestamp = &ts
+ }
parsed, _ := urlutil.ParseAbsoluteURL(input, false)
if parsed != nil {
request.Address = parsed.Hostname()
From 8b3379aa4c18b3b841dc13ca91a4860aa16b10c5 Mon Sep 17 00:00:00 2001
From: Tarun Koyalwar <45962551+tarunKoyalwar@users.noreply.github.com>
Date: Sun, 16 Jun 2024 03:18:01 +0530
Subject: [PATCH 055/283] add team-id env input support (#5295)
---
internal/pdcp/writer.go | 7 +++++++
1 file changed, 7 insertions(+)
diff --git a/internal/pdcp/writer.go b/internal/pdcp/writer.go
index 6e05410355..5dfda78ca2 100644
--- a/internal/pdcp/writer.go
+++ b/internal/pdcp/writer.go
@@ -18,6 +18,7 @@ import (
"github.com/projectdiscovery/nuclei/v3/pkg/output"
"github.com/projectdiscovery/retryablehttp-go"
pdcpauth "github.com/projectdiscovery/utils/auth/pdcp"
+ "github.com/projectdiscovery/utils/env"
errorutil "github.com/projectdiscovery/utils/errors"
unitutils "github.com/projectdiscovery/utils/unit"
updateutils "github.com/projectdiscovery/utils/update"
@@ -30,11 +31,14 @@ const (
flushTimer = time.Minute
MaxChunkSize = 4 * unitutils.Mega // 4 MB
xidRe = `^[a-z0-9]{20}$`
+ teamIDHeader = "X-Team-Id"
)
var (
xidRegex = regexp.MustCompile(xidRe)
_ output.Writer = &UploadWriter{}
+ // teamID if given
+ teamID = env.GetEnvOrDefault("PDCP_TEAM_ID", "")
)
// UploadWriter is a writer that uploads its output to pdcp
@@ -244,6 +248,9 @@ func (u *UploadWriter) getRequest(bin []byte) (*retryablehttp.Request, error) {
req.URL.Update()
req.Header.Set(pdcpauth.ApiKeyHeaderName, u.creds.APIKey)
+ if teamID != "" {
+ req.Header.Set(teamIDHeader, teamID)
+ }
req.Header.Set("Content-Type", "application/octet-stream")
req.Header.Set("Accept", "application/json")
return req, nil
From 9f1faa0a29a64904ecc3b0f4752e5e85ed86f734 Mon Sep 17 00:00:00 2001
From: Dogan Can Bakir <65292895+dogancanbakir@users.noreply.github.com>
Date: Sun, 16 Jun 2024 01:00:21 +0300
Subject: [PATCH 056/283] fix jira nil deref err (#5283)
---
pkg/reporting/trackers/jira/jira.go | 9 +++++----
1 file changed, 5 insertions(+), 4 deletions(-)
diff --git a/pkg/reporting/trackers/jira/jira.go b/pkg/reporting/trackers/jira/jira.go
index 0b8f27fc5b..1fd2895f13 100644
--- a/pkg/reporting/trackers/jira/jira.go
+++ b/pkg/reporting/trackers/jira/jira.go
@@ -16,6 +16,7 @@ import (
"github.com/projectdiscovery/nuclei/v3/pkg/reporting/format"
"github.com/projectdiscovery/nuclei/v3/pkg/reporting/trackers/filters"
"github.com/projectdiscovery/retryablehttp-go"
+ "github.com/projectdiscovery/utils/ptr"
)
type Formatter struct {
@@ -147,13 +148,13 @@ func (i *Integration) CreateNewIssue(event *output.ResultEvent) (*filters.Create
nestedValue = strings.TrimPrefix(fmtNestedValue, "$")
switch nestedValue {
case "CVSSMetrics":
- nestedValue = event.Info.Classification.CVSSMetrics
+ nestedValue = ptr.Safe(event.Info.Classification).CVSSMetrics
case "CVEID":
- nestedValue = event.Info.Classification.CVEID
+ nestedValue = ptr.Safe(event.Info.Classification).CVEID
case "CWEID":
- nestedValue = event.Info.Classification.CWEID
+ nestedValue = ptr.Safe(event.Info.Classification).CWEID
case "CVSSScore":
- nestedValue = event.Info.Classification.CVSSScore
+ nestedValue = ptr.Safe(event.Info.Classification).CVSSScore
case "Host":
nestedValue = event.Host
case "Severity":
From 4720d8c8ec0acf52d2b2cf4d534cbee59323d325 Mon Sep 17 00:00:00 2001
From: map1e
Date: Sun, 16 Jun 2024 06:35:08 +0800
Subject: [PATCH 057/283] fix: types.RequestResponse url field UnmarshalJSON
bug (#5267)
* fix: types.RequestResponse url field UnmarshalJSON bug
* use UnmarshalJSON method in test
* add http unmarshal json test case
---
pkg/input/types/http.go | 8 ++++++--
pkg/input/types/http_test.go | 23 +++++++++++++++++++++++
2 files changed, 29 insertions(+), 2 deletions(-)
diff --git a/pkg/input/types/http.go b/pkg/input/types/http.go
index dbc5c9e325..0620d85f8c 100644
--- a/pkg/input/types/http.go
+++ b/pkg/input/types/http.go
@@ -123,11 +123,15 @@ func (rr *RequestResponse) UnmarshalJSON(data []byte) error {
if err := json.Unmarshal(data, &m); err != nil {
return err
}
- urlStr, ok := m["url"]
+ urlStrRaw, ok := m["url"]
if !ok {
return fmt.Errorf("missing url in request response")
}
- parsed, err := urlutil.ParseAbsoluteURL(string(urlStr), false)
+ var urlStr string
+ if err := json.Unmarshal(urlStrRaw, &urlStr); err != nil {
+ return err
+ }
+ parsed, err := urlutil.ParseAbsoluteURL(urlStr, false)
if err != nil {
return err
}
diff --git a/pkg/input/types/http_test.go b/pkg/input/types/http_test.go
index d674c56353..9525c516e0 100644
--- a/pkg/input/types/http_test.go
+++ b/pkg/input/types/http_test.go
@@ -65,3 +65,26 @@ func TestParseHttpRequest(t *testing.T) {
})
}
}
+
+func TestUnmarshalJSON(t *testing.T) {
+ tests := []struct {
+ name string
+ rawJSONStr string
+ expectedURLStr string
+ }{
+ {"basic url", `{"url": "example.com"}`, "example.com"},
+ {"basic url with scheme", `{"url": "http://example.com"}`, "http://example.com"},
+ }
+ for _, tc := range tests {
+ t.Run(tc.name, func(t *testing.T) {
+ var rr RequestResponse
+ err := rr.UnmarshalJSON([]byte(tc.rawJSONStr))
+ if err != nil {
+ t.Fatal(err)
+ }
+ if tc.expectedURLStr != "" {
+ require.Equal(t, rr.URL.String(), tc.expectedURLStr)
+ }
+ })
+ }
+}
From 0eddafe2a96eb4c098b2abb3a741d625fc219e5a Mon Sep 17 00:00:00 2001
From: Ramana Reddy <90540245+RamanaReddy0M@users.noreply.github.com>
Date: Sun, 16 Jun 2024 04:27:31 +0530
Subject: [PATCH 058/283] Fix tempalte-id validation (#5261)
* Fix tempalte-id validation
* Add checks to mandatory fields
* misc update
---
pkg/templates/templates.go | 12 ++++++++++++
1 file changed, 12 insertions(+)
diff --git a/pkg/templates/templates.go b/pkg/templates/templates.go
index 6403c2767a..045f947c68 100644
--- a/pkg/templates/templates.go
+++ b/pkg/templates/templates.go
@@ -23,6 +23,7 @@ import (
"github.com/projectdiscovery/nuclei/v3/pkg/protocols/websocket"
"github.com/projectdiscovery/nuclei/v3/pkg/protocols/whois"
"github.com/projectdiscovery/nuclei/v3/pkg/templates/types"
+ "github.com/projectdiscovery/nuclei/v3/pkg/utils"
"github.com/projectdiscovery/nuclei/v3/pkg/workflows"
errorutil "github.com/projectdiscovery/utils/errors"
fileutil "github.com/projectdiscovery/utils/file"
@@ -326,6 +327,17 @@ func (template *Template) UnmarshalYAML(unmarshal func(interface{}) error) error
}
*template = Template(*alias)
+ if !ReTemplateID.MatchString(template.ID) {
+ return errorutil.New("template id must match expression %v", ReTemplateID).WithTag("invalid template")
+ }
+ info := template.Info
+ if utils.IsBlank(info.Name) {
+ return errorutil.New("no template name field provided").WithTag("invalid template")
+ }
+ if info.Authors.IsEmpty() {
+ return errorutil.New("no template author field provided").WithTag("invalid template")
+ }
+
if len(template.RequestsHTTP) > 0 || len(template.RequestsNetwork) > 0 {
_ = deprecatedProtocolNameTemplates.Set(template.ID, true)
}
From b274fe5ed126a06f72e6ca6b797296edb5f9e211 Mon Sep 17 00:00:00 2001
From: sandeep <8293321+ehsandeep@users.noreply.github.com>
Date: Sun, 16 Jun 2024 04:47:59 +0530
Subject: [PATCH 059/283] version update
---
pkg/catalog/config/constants.go | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/pkg/catalog/config/constants.go b/pkg/catalog/config/constants.go
index bfd95f1b88..0ee9f437bb 100644
--- a/pkg/catalog/config/constants.go
+++ b/pkg/catalog/config/constants.go
@@ -31,7 +31,7 @@ const (
CLIConfigFileName = "config.yaml"
ReportingConfigFilename = "reporting-config.yaml"
// Version is the current version of nuclei
- Version = `v3.2.8`
+ Version = `v3.2.9`
// Directory Names of custom templates
CustomS3TemplatesDirName = "s3"
CustomGitHubTemplatesDirName = "github"
From f8842b1f04312a69de0fbbcbd7faa03b63c14e4b Mon Sep 17 00:00:00 2001
From: Ice3man
Date: Sun, 16 Jun 2024 19:14:43 +0530
Subject: [PATCH 060/283] fix: fixed individual per tracker reporting filters
(#5297)
* fix: fixed individual per tracker reporting filters
* added test case
---
pkg/reporting/reporting.go | 2 +-
pkg/reporting/trackers/gitea/gitea.go | 6 ++---
pkg/reporting/trackers/github/github.go | 6 ++---
pkg/reporting/trackers/gitlab/gitlab.go | 6 ++---
pkg/reporting/trackers/jira/jira.go | 6 ++---
pkg/reporting/trackers/jira/jira_test.go | 34 ++++++++++++++++++++++++
6 files changed, 47 insertions(+), 13 deletions(-)
diff --git a/pkg/reporting/reporting.go b/pkg/reporting/reporting.go
index 08f32e9313..172c31105e 100644
--- a/pkg/reporting/reporting.go
+++ b/pkg/reporting/reporting.go
@@ -273,7 +273,7 @@ func (c *ReportingClient) CreateIssue(event *output.ResultEvent) error {
for _, tracker := range c.trackers {
// process tracker specific allow/deny list
- if tracker.ShouldFilter(event) {
+ if !tracker.ShouldFilter(event) {
continue
}
diff --git a/pkg/reporting/trackers/gitea/gitea.go b/pkg/reporting/trackers/gitea/gitea.go
index acb82a1de2..e433689c74 100644
--- a/pkg/reporting/trackers/gitea/gitea.go
+++ b/pkg/reporting/trackers/gitea/gitea.go
@@ -139,15 +139,15 @@ func (i *Integration) CloseIssue(event *output.ResultEvent) error {
// ShouldFilter determines if an issue should be logged to this tracker
func (i *Integration) ShouldFilter(event *output.ResultEvent) bool {
- if i.options.AllowList != nil && i.options.AllowList.GetMatch(event) {
+ if i.options.AllowList != nil && !i.options.AllowList.GetMatch(event) {
return false
}
if i.options.DenyList != nil && i.options.DenyList.GetMatch(event) {
- return true
+ return false
}
- return false
+ return true
}
func (i *Integration) findIssueByTitle(title string) (*gitea.Issue, error) {
diff --git a/pkg/reporting/trackers/github/github.go b/pkg/reporting/trackers/github/github.go
index 44c06102ca..8011d8da53 100644
--- a/pkg/reporting/trackers/github/github.go
+++ b/pkg/reporting/trackers/github/github.go
@@ -175,15 +175,15 @@ func (i *Integration) Name() string {
// ShouldFilter determines if an issue should be logged to this tracker
func (i *Integration) ShouldFilter(event *output.ResultEvent) bool {
- if i.options.AllowList != nil && i.options.AllowList.GetMatch(event) {
+ if i.options.AllowList != nil && !i.options.AllowList.GetMatch(event) {
return false
}
if i.options.DenyList != nil && i.options.DenyList.GetMatch(event) {
- return true
+ return false
}
- return false
+ return true
}
func (i *Integration) findIssueByTitle(ctx context.Context, title string) (*github.Issue, error) {
diff --git a/pkg/reporting/trackers/gitlab/gitlab.go b/pkg/reporting/trackers/gitlab/gitlab.go
index 769165ad48..d816a26af6 100644
--- a/pkg/reporting/trackers/gitlab/gitlab.go
+++ b/pkg/reporting/trackers/gitlab/gitlab.go
@@ -164,13 +164,13 @@ func (i *Integration) CloseIssue(event *output.ResultEvent) error {
// ShouldFilter determines if an issue should be logged to this tracker
func (i *Integration) ShouldFilter(event *output.ResultEvent) bool {
- if i.options.AllowList != nil && i.options.AllowList.GetMatch(event) {
+ if i.options.AllowList != nil && !i.options.AllowList.GetMatch(event) {
return false
}
if i.options.DenyList != nil && i.options.DenyList.GetMatch(event) {
- return true
+ return false
}
- return false
+ return true
}
diff --git a/pkg/reporting/trackers/jira/jira.go b/pkg/reporting/trackers/jira/jira.go
index 1fd2895f13..5499445c87 100644
--- a/pkg/reporting/trackers/jira/jira.go
+++ b/pkg/reporting/trackers/jira/jira.go
@@ -315,13 +315,13 @@ func (i *Integration) FindExistingIssue(event *output.ResultEvent) (jira.Issue,
// ShouldFilter determines if an issue should be logged to this tracker
func (i *Integration) ShouldFilter(event *output.ResultEvent) bool {
- if i.options.AllowList != nil && i.options.AllowList.GetMatch(event) {
+ if i.options.AllowList != nil && !i.options.AllowList.GetMatch(event) {
return false
}
if i.options.DenyList != nil && i.options.DenyList.GetMatch(event) {
- return true
+ return false
}
- return false
+ return true
}
diff --git a/pkg/reporting/trackers/jira/jira_test.go b/pkg/reporting/trackers/jira/jira_test.go
index 247a440c58..d725a97b29 100644
--- a/pkg/reporting/trackers/jira/jira_test.go
+++ b/pkg/reporting/trackers/jira/jira_test.go
@@ -4,6 +4,10 @@ import (
"strings"
"testing"
+ "github.com/projectdiscovery/nuclei/v3/pkg/model"
+ "github.com/projectdiscovery/nuclei/v3/pkg/model/types/severity"
+ "github.com/projectdiscovery/nuclei/v3/pkg/output"
+ "github.com/projectdiscovery/nuclei/v3/pkg/reporting/trackers/filters"
"github.com/stretchr/testify/require"
)
@@ -36,3 +40,33 @@ func TestTableCreation(t *testing.T) {
`
require.Equal(t, expected, table)
}
+
+func Test_ShouldFilter_Tracker(t *testing.T) {
+ jiraIntegration := &Integration{
+ options: &Options{AllowList: &filters.Filter{
+ Severities: severity.Severities{severity.Critical},
+ }},
+ }
+
+ require.False(t, jiraIntegration.ShouldFilter(&output.ResultEvent{Info: model.Info{
+ SeverityHolder: severity.Holder{Severity: severity.Info},
+ }}))
+ require.True(t, jiraIntegration.ShouldFilter(&output.ResultEvent{Info: model.Info{
+ SeverityHolder: severity.Holder{Severity: severity.Critical},
+ }}))
+
+ t.Run("deny-list", func(t *testing.T) {
+ jiraIntegration := &Integration{
+ options: &Options{DenyList: &filters.Filter{
+ Severities: severity.Severities{severity.Critical},
+ }},
+ }
+
+ require.True(t, jiraIntegration.ShouldFilter(&output.ResultEvent{Info: model.Info{
+ SeverityHolder: severity.Holder{Severity: severity.Info},
+ }}))
+ require.False(t, jiraIntegration.ShouldFilter(&output.ResultEvent{Info: model.Info{
+ SeverityHolder: severity.Holder{Severity: severity.Critical},
+ }}))
+ })
+}
From 930f9ab02d5605f5d834804e52fdb308e1776acf Mon Sep 17 00:00:00 2001
From: sandeep <8293321+ehsandeep@users.noreply.github.com>
Date: Sun, 16 Jun 2024 18:57:58 +0400
Subject: [PATCH 061/283] build workflow update
---
.github/workflows/release-binary.yml | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/.github/workflows/release-binary.yml b/.github/workflows/release-binary.yml
index 9622650d1f..5c7181776a 100644
--- a/.github/workflows/release-binary.yml
+++ b/.github/workflows/release-binary.yml
@@ -21,7 +21,7 @@ jobs:
- uses: goreleaser/goreleaser-action@v4
with:
- args: "release --rm-dist"
+ args: "release --clean"
version: latest
env:
GITHUB_TOKEN: "${{ secrets.GITHUB_TOKEN }}"
From c175474df51bb08e5fb53d82a334d5ee11b76125 Mon Sep 17 00:00:00 2001
From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com>
Date: Mon, 17 Jun 2024 05:57:05 +0000
Subject: [PATCH 062/283] chore(deps): bump github.com/projectdiscovery/goflags
Bumps [github.com/projectdiscovery/goflags](https://github.com/projectdiscovery/goflags) from 0.1.54 to 0.1.56.
- [Release notes](https://github.com/projectdiscovery/goflags/releases)
- [Commits](https://github.com/projectdiscovery/goflags/compare/v0.1.54...v0.1.56)
---
updated-dependencies:
- dependency-name: github.com/projectdiscovery/goflags
dependency-type: direct:production
update-type: version-update:semver-patch
...
Signed-off-by: dependabot[bot]
---
go.mod | 2 +-
go.sum | 4 ++--
2 files changed, 3 insertions(+), 3 deletions(-)
diff --git a/go.mod b/go.mod
index 6970a332a2..26fd609619 100644
--- a/go.mod
+++ b/go.mod
@@ -81,7 +81,7 @@ require (
github.com/projectdiscovery/dsl v0.1.2
github.com/projectdiscovery/fasttemplate v0.0.2
github.com/projectdiscovery/go-smb2 v0.0.0-20240129202741-052cc450c6cb
- github.com/projectdiscovery/goflags v0.1.54
+ github.com/projectdiscovery/goflags v0.1.56
github.com/projectdiscovery/gologger v1.1.12
github.com/projectdiscovery/gostruct v0.0.2
github.com/projectdiscovery/gozero v0.0.2
diff --git a/go.sum b/go.sum
index a3702b0fdb..d8c1f3d29f 100644
--- a/go.sum
+++ b/go.sum
@@ -841,8 +841,8 @@ github.com/projectdiscovery/freeport v0.0.5 h1:jnd3Oqsl4S8n0KuFkE5Hm8WGDP24ITBvm
github.com/projectdiscovery/freeport v0.0.5/go.mod h1:PY0bxSJ34HVy67LHIeF3uIutiCSDwOqKD8ruBkdiCwE=
github.com/projectdiscovery/go-smb2 v0.0.0-20240129202741-052cc450c6cb h1:rutG906Drtbpz4DwU5mhGIeOhRcktDH4cGQitGUMAsg=
github.com/projectdiscovery/go-smb2 v0.0.0-20240129202741-052cc450c6cb/go.mod h1:FLjF1DmZ+POoGEiIQdWuYVwS++C/GwpX8YaCsTSm1RY=
-github.com/projectdiscovery/goflags v0.1.54 h1:xbaEBNbKqXyRu4154UlhsMAFmpaDYma4jJ9zUZGvXW8=
-github.com/projectdiscovery/goflags v0.1.54/go.mod h1:cfLzWWxgl/ft5cSHVJZjnvIzd4wX3A9Kz+W1pjeBZvc=
+github.com/projectdiscovery/goflags v0.1.56 h1:tJYiZN7s9Jk9DxfYOUiqOoybaIDlXyX4ZgT4B/06SyU=
+github.com/projectdiscovery/goflags v0.1.56/go.mod h1:DsGF0NPpM5hGg75N3MTSvWJ4MIT7HFEAOEeWZ074+Fg=
github.com/projectdiscovery/gologger v1.1.12 h1:uX/QkQdip4PubJjjG0+uk5DtyAi1ANPJUvpmimXqv4A=
github.com/projectdiscovery/gologger v1.1.12/go.mod h1:DI8nywPLERS5mo8QEA9E7gd5HZ3Je14SjJBH3F5/kLw=
github.com/projectdiscovery/gostruct v0.0.2 h1:s8gP8ApugGM4go1pA+sVlPDXaWqNP5BBDDSv7VEdG1M=
From d7d26ca9b5221f09a3f6893dffde3990431b070d Mon Sep 17 00:00:00 2001
From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com>
Date: Mon, 17 Jun 2024 05:57:14 +0000
Subject: [PATCH 063/283] chore(deps): bump
github.com/projectdiscovery/wappalyzergo
Bumps [github.com/projectdiscovery/wappalyzergo](https://github.com/projectdiscovery/wappalyzergo) from 0.1.4 to 0.1.6.
- [Release notes](https://github.com/projectdiscovery/wappalyzergo/releases)
- [Commits](https://github.com/projectdiscovery/wappalyzergo/compare/v0.1.4...v0.1.6)
---
updated-dependencies:
- dependency-name: github.com/projectdiscovery/wappalyzergo
dependency-type: direct:production
update-type: version-update:semver-patch
...
Signed-off-by: dependabot[bot]
---
go.mod | 2 +-
go.sum | 4 ++--
2 files changed, 3 insertions(+), 3 deletions(-)
diff --git a/go.mod b/go.mod
index 6970a332a2..6ac096b84b 100644
--- a/go.mod
+++ b/go.mod
@@ -95,7 +95,7 @@ require (
github.com/projectdiscovery/uncover v1.0.8
github.com/projectdiscovery/useragent v0.0.54
github.com/projectdiscovery/utils v0.1.4-0.20240611113448-0e2f2d33fe1c
- github.com/projectdiscovery/wappalyzergo v0.1.4
+ github.com/projectdiscovery/wappalyzergo v0.1.6
github.com/redis/go-redis/v9 v9.1.0
github.com/seh-msft/burpxml v1.0.1
github.com/stretchr/testify v1.9.0
diff --git a/go.sum b/go.sum
index a3702b0fdb..18f6dbac79 100644
--- a/go.sum
+++ b/go.sum
@@ -887,8 +887,8 @@ github.com/projectdiscovery/useragent v0.0.54 h1:3gM4TnhLSspSM/XkI97YcUazXA3KOkQ
github.com/projectdiscovery/useragent v0.0.54/go.mod h1:dKFkqUb44qx7MpwQDAbhL1t2IrFhcrgI5MlXUZ9E8dM=
github.com/projectdiscovery/utils v0.1.4-0.20240611113448-0e2f2d33fe1c h1:0I/iRtu5nPYle1v8/R33pCLOrH5bziP5Bi0eZURxTQY=
github.com/projectdiscovery/utils v0.1.4-0.20240611113448-0e2f2d33fe1c/go.mod h1:mXs6OOeG9l/dVchjB2PGvQO3+wuMiE14Y/kmHeKogoM=
-github.com/projectdiscovery/wappalyzergo v0.1.4 h1:Q0j5t/c74DlPKRvGYLZFv8npTA7MBW9uwdlsMBn/Yh4=
-github.com/projectdiscovery/wappalyzergo v0.1.4/go.mod h1:wBYGKmA5BQp/NWsAy1q/jSH8N1LHWQ/LV26DuR+KzPM=
+github.com/projectdiscovery/wappalyzergo v0.1.6 h1:4iZDEnytsnAqjQgD5K6lmAzVx++KSQeMCDfomDtQM1M=
+github.com/projectdiscovery/wappalyzergo v0.1.6/go.mod h1:wBYGKmA5BQp/NWsAy1q/jSH8N1LHWQ/LV26DuR+KzPM=
github.com/projectdiscovery/yamldoc-go v1.0.4 h1:eZoESapnMw6WAHiVgRwNqvbJEfNHEH148uthhFbG5jE=
github.com/projectdiscovery/yamldoc-go v1.0.4/go.mod h1:8PIPRcUD55UbtQdcfFR1hpIGRWG0P7alClXNGt1TBik=
github.com/prometheus/client_golang v0.9.1/go.mod h1:7SWBe2y4D6OKWSNQJUaRYU/AaXPKyh/dDVn+NZz0KFw=
From 1359c2e7b9ade47af8a8be0fca5e9fe4b8d87929 Mon Sep 17 00:00:00 2001
From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com>
Date: Mon, 17 Jun 2024 06:25:00 +0000
Subject: [PATCH 064/283] chore(deps): bump
github.com/projectdiscovery/useragent
Bumps [github.com/projectdiscovery/useragent](https://github.com/projectdiscovery/useragent) from 0.0.54 to 0.0.56.
- [Release notes](https://github.com/projectdiscovery/useragent/releases)
- [Commits](https://github.com/projectdiscovery/useragent/compare/v0.0.54...v0.0.56)
---
updated-dependencies:
- dependency-name: github.com/projectdiscovery/useragent
dependency-type: direct:production
update-type: version-update:semver-patch
...
Signed-off-by: dependabot[bot]
---
go.mod | 2 +-
go.sum | 4 ++--
2 files changed, 3 insertions(+), 3 deletions(-)
diff --git a/go.mod b/go.mod
index 4584f83646..f95945df2b 100644
--- a/go.mod
+++ b/go.mod
@@ -93,7 +93,7 @@ require (
github.com/projectdiscovery/sarif v0.0.1
github.com/projectdiscovery/tlsx v1.1.6
github.com/projectdiscovery/uncover v1.0.8
- github.com/projectdiscovery/useragent v0.0.54
+ github.com/projectdiscovery/useragent v0.0.56
github.com/projectdiscovery/utils v0.1.4-0.20240611113448-0e2f2d33fe1c
github.com/projectdiscovery/wappalyzergo v0.1.6
github.com/redis/go-redis/v9 v9.1.0
diff --git a/go.sum b/go.sum
index efaa306f61..078beeb61a 100644
--- a/go.sum
+++ b/go.sum
@@ -883,8 +883,8 @@ github.com/projectdiscovery/tlsx v1.1.6 h1:iw2zwKbd2+kRQ8J1G4dLmS0CLyemd/tKz1Uzc
github.com/projectdiscovery/tlsx v1.1.6/go.mod h1:s7SRRFdrwIZBK/RXXZi4CR/CubqFSvp8h5Bk1srEZIo=
github.com/projectdiscovery/uncover v1.0.8 h1:nE8bYJuwhqk0BEMRQRhNbjPGMy40A9gkiMk5xq6U4u0=
github.com/projectdiscovery/uncover v1.0.8/go.mod h1:1SwsNqjwMkJEzJQ7lQr5AHIdYd+BQlyqhO5IwIYmIAM=
-github.com/projectdiscovery/useragent v0.0.54 h1:3gM4TnhLSspSM/XkI97YcUazXA3KOkQXEU7TIBly2Fc=
-github.com/projectdiscovery/useragent v0.0.54/go.mod h1:dKFkqUb44qx7MpwQDAbhL1t2IrFhcrgI5MlXUZ9E8dM=
+github.com/projectdiscovery/useragent v0.0.56 h1:KpUtOgq1vUerPrgRFZAIr1hF0lDxA2RF6YkT2PLwSoQ=
+github.com/projectdiscovery/useragent v0.0.56/go.mod h1:LAh9vudxXPA5JfNN3+1g/Unqvulq0BDkjxK7IrE+e/s=
github.com/projectdiscovery/utils v0.1.4-0.20240611113448-0e2f2d33fe1c h1:0I/iRtu5nPYle1v8/R33pCLOrH5bziP5Bi0eZURxTQY=
github.com/projectdiscovery/utils v0.1.4-0.20240611113448-0e2f2d33fe1c/go.mod h1:mXs6OOeG9l/dVchjB2PGvQO3+wuMiE14Y/kmHeKogoM=
github.com/projectdiscovery/wappalyzergo v0.1.6 h1:4iZDEnytsnAqjQgD5K6lmAzVx++KSQeMCDfomDtQM1M=
From 20709834c2679b1bc15d28efc17546291dccf36a Mon Sep 17 00:00:00 2001
From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com>
Date: Mon, 17 Jun 2024 06:25:20 +0000
Subject: [PATCH 065/283] chore(deps): bump github.com/projectdiscovery/dsl
from 0.1.2 to 0.1.3
Bumps [github.com/projectdiscovery/dsl](https://github.com/projectdiscovery/dsl) from 0.1.2 to 0.1.3.
- [Release notes](https://github.com/projectdiscovery/dsl/releases)
- [Commits](https://github.com/projectdiscovery/dsl/compare/v0.1.2...v0.1.3)
---
updated-dependencies:
- dependency-name: github.com/projectdiscovery/dsl
dependency-type: direct:production
update-type: version-update:semver-patch
...
Signed-off-by: dependabot[bot]
---
go.mod | 2 +-
go.sum | 4 ++--
2 files changed, 3 insertions(+), 3 deletions(-)
diff --git a/go.mod b/go.mod
index 4584f83646..a072f5cbfd 100644
--- a/go.mod
+++ b/go.mod
@@ -78,7 +78,7 @@ require (
github.com/mholt/archiver v3.1.1+incompatible
github.com/ory/dockertest/v3 v3.10.0
github.com/praetorian-inc/fingerprintx v1.1.9
- github.com/projectdiscovery/dsl v0.1.2
+ github.com/projectdiscovery/dsl v0.1.3
github.com/projectdiscovery/fasttemplate v0.0.2
github.com/projectdiscovery/go-smb2 v0.0.0-20240129202741-052cc450c6cb
github.com/projectdiscovery/goflags v0.1.56
diff --git a/go.sum b/go.sum
index efaa306f61..dd2a0025cb 100644
--- a/go.sum
+++ b/go.sum
@@ -831,8 +831,8 @@ github.com/projectdiscovery/cdncheck v1.0.9 h1:BS15gzj9gb5AVSKqTDzPamfSgStu7nJQO
github.com/projectdiscovery/cdncheck v1.0.9/go.mod h1:18SSl1w7rMj53CGeRIZTbDoa286a6xZIxGbaiEo4Fxs=
github.com/projectdiscovery/clistats v0.0.20 h1:5jO5SLiRJ7f0nDV0ndBNmBeesbROouPooH+DGMgoWq4=
github.com/projectdiscovery/clistats v0.0.20/go.mod h1:GJ2av0KnOvK0AISQnP8hyDclYIji1LVkx2l0pwnzAu4=
-github.com/projectdiscovery/dsl v0.1.2 h1:RdWCEsUqyWsLIQ1k16NX5FiIVbPFvzx9BOlvzK7eeGw=
-github.com/projectdiscovery/dsl v0.1.2/go.mod h1:3zwogxGPnPkVUk5OfZQ9Y51sAWdazfQoVZFSPqeEUYA=
+github.com/projectdiscovery/dsl v0.1.3 h1:Lxlgpx8lwq/CaBjVUmwF6KmkMDnYf9IVKlMsWCF72OE=
+github.com/projectdiscovery/dsl v0.1.3/go.mod h1:WAuQLD4KQ/94RASotE7cvk7X6kY7JQG9Fcqc3lRQfpI=
github.com/projectdiscovery/fastdialer v0.1.4-0.20240611222741-427bffb2ea13 h1:mAehxpgxmvFrKcQ8taA3xQskp3gg6GayAgatTAn37bg=
github.com/projectdiscovery/fastdialer v0.1.4-0.20240611222741-427bffb2ea13/go.mod h1:PcNscfCNrCnrkmvob90yfd+rNDMh6ZOIvRIl6HFPPXE=
github.com/projectdiscovery/fasttemplate v0.0.2 h1:h2cISk5xDhlJEinlBQS6RRx0vOlOirB2y3Yu4PJzpiA=
From c16050a5fe3977a7948aec460d4dd6163051ad65 Mon Sep 17 00:00:00 2001
From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com>
Date: Mon, 17 Jun 2024 06:54:38 +0000
Subject: [PATCH 066/283] chore(deps): bump github.com/projectdiscovery/utils
Bumps [github.com/projectdiscovery/utils](https://github.com/projectdiscovery/utils) from 0.1.4-0.20240611113448-0e2f2d33fe1c to 0.1.4.
- [Release notes](https://github.com/projectdiscovery/utils/releases)
- [Changelog](https://github.com/projectdiscovery/utils/blob/main/CHANGELOG.md)
- [Commits](https://github.com/projectdiscovery/utils/commits/v0.1.4)
---
updated-dependencies:
- dependency-name: github.com/projectdiscovery/utils
dependency-type: direct:production
update-type: version-update:semver-patch
...
Signed-off-by: dependabot[bot]
---
go.mod | 2 +-
go.sum | 4 ++--
2 files changed, 3 insertions(+), 3 deletions(-)
diff --git a/go.mod b/go.mod
index fb8e067fea..27a4f2b734 100644
--- a/go.mod
+++ b/go.mod
@@ -94,7 +94,7 @@ require (
github.com/projectdiscovery/tlsx v1.1.6
github.com/projectdiscovery/uncover v1.0.8
github.com/projectdiscovery/useragent v0.0.56
- github.com/projectdiscovery/utils v0.1.4-0.20240611113448-0e2f2d33fe1c
+ github.com/projectdiscovery/utils v0.1.4
github.com/projectdiscovery/wappalyzergo v0.1.6
github.com/redis/go-redis/v9 v9.1.0
github.com/seh-msft/burpxml v1.0.1
diff --git a/go.sum b/go.sum
index 5773d2d4a1..2d9878b4ca 100644
--- a/go.sum
+++ b/go.sum
@@ -885,8 +885,8 @@ github.com/projectdiscovery/uncover v1.0.8 h1:nE8bYJuwhqk0BEMRQRhNbjPGMy40A9gkiM
github.com/projectdiscovery/uncover v1.0.8/go.mod h1:1SwsNqjwMkJEzJQ7lQr5AHIdYd+BQlyqhO5IwIYmIAM=
github.com/projectdiscovery/useragent v0.0.56 h1:KpUtOgq1vUerPrgRFZAIr1hF0lDxA2RF6YkT2PLwSoQ=
github.com/projectdiscovery/useragent v0.0.56/go.mod h1:LAh9vudxXPA5JfNN3+1g/Unqvulq0BDkjxK7IrE+e/s=
-github.com/projectdiscovery/utils v0.1.4-0.20240611113448-0e2f2d33fe1c h1:0I/iRtu5nPYle1v8/R33pCLOrH5bziP5Bi0eZURxTQY=
-github.com/projectdiscovery/utils v0.1.4-0.20240611113448-0e2f2d33fe1c/go.mod h1:mXs6OOeG9l/dVchjB2PGvQO3+wuMiE14Y/kmHeKogoM=
+github.com/projectdiscovery/utils v0.1.4 h1:VSULN/RBEXbkgWuhrDey84ZDnSXtYMSev9p+QHXXDeI=
+github.com/projectdiscovery/utils v0.1.4/go.mod h1:mXs6OOeG9l/dVchjB2PGvQO3+wuMiE14Y/kmHeKogoM=
github.com/projectdiscovery/wappalyzergo v0.1.6 h1:4iZDEnytsnAqjQgD5K6lmAzVx++KSQeMCDfomDtQM1M=
github.com/projectdiscovery/wappalyzergo v0.1.6/go.mod h1:wBYGKmA5BQp/NWsAy1q/jSH8N1LHWQ/LV26DuR+KzPM=
github.com/projectdiscovery/yamldoc-go v1.0.4 h1:eZoESapnMw6WAHiVgRwNqvbJEfNHEH148uthhFbG5jE=
From 1c51a6bef6a5d9a5b9c4967e4984595ef4bb48cd Mon Sep 17 00:00:00 2001
From: Mzack9999
Date: Sat, 22 Jun 2024 19:12:00 +0200
Subject: [PATCH 067/283] improving workflows (#5318)
* improving workflows
* fixing count
---
pkg/templates/compile.go | 5 +++++
pkg/templates/compile_test.go | 9 +++++++++
pkg/templates/templates.go | 4 ++--
pkg/templates/tests/workflow-invalid.yaml | 18 ++++++++++++++++++
4 files changed, 34 insertions(+), 2 deletions(-)
create mode 100644 pkg/templates/tests/workflow-invalid.yaml
diff --git a/pkg/templates/compile.go b/pkg/templates/compile.go
index 39d6a6ea0f..54708e7ea1 100644
--- a/pkg/templates/compile.go
+++ b/pkg/templates/compile.go
@@ -342,6 +342,11 @@ func parseTemplate(data []byte, options protocols.ExecutorOptions) (*Template, e
return nil, errors.New("no template author field provided")
}
+ numberOfWorkflows := len(template.Workflows)
+ if numberOfWorkflows > 0 && numberOfWorkflows != template.Requests() {
+ return nil, errors.New("workflows cannot have other protocols")
+ }
+
// use default unknown severity
if len(template.Workflows) == 0 {
if template.Info.SeverityHolder.Severity == severity.Undefined {
diff --git a/pkg/templates/compile_test.go b/pkg/templates/compile_test.go
index 3a230cc527..81c5f8fd51 100644
--- a/pkg/templates/compile_test.go
+++ b/pkg/templates/compile_test.go
@@ -197,3 +197,12 @@ func Test_WrongTemplate(t *testing.T) {
require.Nil(t, got, "could not parse template")
require.ErrorContains(t, err, "no requests defined ")
}
+
+func TestWrongWorkflow(t *testing.T) {
+ setup()
+
+ filePath := "tests/workflow-invalid.yaml"
+ got, err := templates.Parse(filePath, nil, executerOpts)
+ require.Nil(t, got, "could not parse template")
+ require.ErrorContains(t, err, "workflows cannot have other protocols")
+}
diff --git a/pkg/templates/templates.go b/pkg/templates/templates.go
index 045f947c68..df0c47648d 100644
--- a/pkg/templates/templates.go
+++ b/pkg/templates/templates.go
@@ -177,8 +177,6 @@ func (template *Template) Type() types.ProtocolType {
return types.HeadlessProtocol
case len(template.RequestsNetwork) > 0:
return types.NetworkProtocol
- case len(template.Workflow.Workflows) > 0:
- return types.WorkflowProtocol
case len(template.RequestsSSL) > 0:
return types.SSLProtocol
case len(template.RequestsWebsocket) > 0:
@@ -189,6 +187,8 @@ func (template *Template) Type() types.ProtocolType {
return types.CodeProtocol
case len(template.RequestsJavascript) > 0:
return types.JavascriptProtocol
+ case len(template.Workflow.Workflows) > 0:
+ return types.WorkflowProtocol
default:
return types.InvalidProtocol
}
diff --git a/pkg/templates/tests/workflow-invalid.yaml b/pkg/templates/tests/workflow-invalid.yaml
new file mode 100644
index 0000000000..981823aa0c
--- /dev/null
+++ b/pkg/templates/tests/workflow-invalid.yaml
@@ -0,0 +1,18 @@
+id: workflow-example
+
+info:
+ name: Test Invalid Workflow Template
+ author: pdteam
+ severity: info
+
+http:
+ - raw:
+ - |
+ POST /re HTTP/1.1
+ Host: {{Hostname}}
+
+ {{code_response}}
+
+workflows:
+ - template: tests/match-1.yaml
+ - template: tests/match-1.yaml
From e61ca0c42d3e51b2230f84b05ccc618b0f4b0d2d Mon Sep 17 00:00:00 2001
From: Douglas Danger Manley
<127235272+doug-threatmate@users.noreply.github.com>
Date: Sat, 22 Jun 2024 15:42:01 -0400
Subject: [PATCH 068/283] Use `fs.FS` when explicitly given (#5312)
* Use the `templateFS` if it's there when calling `OpenFile`
* Add a new constructor
* More refactoring
* Both of my use cases are working
* Fix for legacy assumptions
* minor update: remove gologger debug stmts
---------
Co-authored-by: Tarun Koyalwar
---
lib/sdk_private.go | 5 +-
pkg/catalog/disk/catalog.go | 32 +++++---
pkg/catalog/disk/find.go | 147 +++++++++++++++++++++++++++---------
3 files changed, 136 insertions(+), 48 deletions(-)
diff --git a/lib/sdk_private.go b/lib/sdk_private.go
index 3475200bd0..b65ecf2a2f 100644
--- a/lib/sdk_private.go
+++ b/lib/sdk_private.go
@@ -231,7 +231,10 @@ func (e *NucleiEngine) init(ctx context.Context) error {
// and also upgrade templates to latest version if available
installer.NucleiSDKVersionCheck()
- return e.processUpdateCheckResults()
+ if DefaultConfig.CanCheckForUpdates() {
+ return e.processUpdateCheckResults()
+ }
+ return nil
}
type syncOnce struct {
diff --git a/pkg/catalog/disk/catalog.go b/pkg/catalog/disk/catalog.go
index ce1a080fdd..38c2260758 100644
--- a/pkg/catalog/disk/catalog.go
+++ b/pkg/catalog/disk/catalog.go
@@ -11,17 +11,25 @@ import (
// DiskCatalog is a template catalog helper implementation based on disk
type DiskCatalog struct {
templatesDirectory string
- templatesFS fs.FS // TODO: Refactor to use this
+ templatesFS fs.FS // Due to issues with how Go has implemented fs.FS, we'll have to also implement normal os operations, as well. See: https://github.com/golang/go/issues/44279
}
// NewCatalog creates a new Catalog structure using provided input items
// using disk based items
func NewCatalog(directory string) *DiskCatalog {
catalog := &DiskCatalog{templatesDirectory: directory}
- if directory != "" {
- catalog.templatesFS = os.DirFS(directory)
- } else {
- catalog.templatesFS = os.DirFS(config.DefaultConfig.GetTemplateDir())
+ if directory == "" {
+ catalog.templatesDirectory = config.DefaultConfig.GetTemplateDir()
+ }
+ return catalog
+}
+
+// NewFSCatalog creates a new Catalog structure using provided input items
+// using the fs.FS as its filesystem.
+func NewFSCatalog(fs fs.FS, directory string) *DiskCatalog {
+ catalog := &DiskCatalog{
+ templatesDirectory: directory,
+ templatesFS: fs,
}
return catalog
}
@@ -29,11 +37,15 @@ func NewCatalog(directory string) *DiskCatalog {
// OpenFile opens a file and returns an io.ReadCloser to the file.
// It is used to read template and payload files based on catalog responses.
func (d *DiskCatalog) OpenFile(filename string) (io.ReadCloser, error) {
- file, err := os.Open(filename)
- if err != nil {
- if file, errx := os.Open(BackwardsCompatiblePaths(d.templatesDirectory, filename)); errx == nil {
- return file, nil
+ if d.templatesFS == nil {
+ file, err := os.Open(filename)
+ if err != nil {
+ if file, errx := os.Open(BackwardsCompatiblePaths(d.templatesDirectory, filename)); errx == nil {
+ return file, nil
+ }
}
+ return file, err
}
- return file, err
+
+ return d.templatesFS.Open(filename)
}
diff --git a/pkg/catalog/disk/find.go b/pkg/catalog/disk/find.go
index 088bcbfe1f..a5d46ff078 100644
--- a/pkg/catalog/disk/find.go
+++ b/pkg/catalog/disk/find.go
@@ -79,17 +79,21 @@ func (c *DiskCatalog) GetTemplatePath(target string) ([]string, error) {
}
// try to handle deprecated template paths
- absPath := BackwardsCompatiblePaths(c.templatesDirectory, target)
- if absPath != target && strings.TrimPrefix(absPath, c.templatesDirectory+string(filepath.Separator)) != target {
- if config.DefaultConfig.LogAllEvents {
- gologger.DefaultLogger.Print().Msgf("[%v] requested Template path %s is deprecated, please update to %s\n", aurora.Yellow("WRN").String(), target, absPath)
+ absPath := target
+ if c.templatesFS == nil {
+ absPath = BackwardsCompatiblePaths(c.templatesDirectory, target)
+ if absPath != target && strings.TrimPrefix(absPath, c.templatesDirectory+string(filepath.Separator)) != target {
+ if config.DefaultConfig.LogAllEvents {
+ gologger.DefaultLogger.Print().Msgf("[%v] requested Template path %s is deprecated, please update to %s\n", aurora.Yellow("WRN").String(), target, absPath)
+ }
+ deprecatedPathsCounter++
}
- deprecatedPathsCounter++
- }
- absPath, err := c.convertPathToAbsolute(absPath)
- if err != nil {
- return nil, errors.Wrapf(err, "could not find template file")
+ var err error
+ absPath, err = c.convertPathToAbsolute(absPath)
+ if err != nil {
+ return nil, errors.Wrapf(err, "could not find template file")
+ }
}
// Template input is either a file or a directory
@@ -143,24 +147,60 @@ func (c *DiskCatalog) findGlobPathMatches(absPath string, processed map[string]s
if c.templatesDirectory == "" {
templateDir = "./"
}
- matches, _ := fs.Glob(os.DirFS(filepath.Join(templateDir, "http")), inputGlob)
- if len(matches) != 0 {
+
+ if c.templatesFS == nil {
+ matches, _ := fs.Glob(os.DirFS(filepath.Join(templateDir, "http")), inputGlob)
+ if len(matches) != 0 {
+ return matches
+ }
+
+ // condition to support network cve related globs
+ matches, _ = fs.Glob(os.DirFS(filepath.Join(templateDir, "network")), inputGlob)
+ return matches
+ } else {
+ sub, err := fs.Sub(c.templatesFS, filepath.Join(templateDir, "http"))
+ if err != nil {
+ return nil
+ }
+ matches, _ := fs.Glob(sub, inputGlob)
+ if len(matches) != 0 {
+ return matches
+ }
+
+ // condition to support network cve related globs
+ sub, err = fs.Sub(c.templatesFS, filepath.Join(templateDir, "network"))
+ if err != nil {
+ return nil
+ }
+ matches, _ = fs.Glob(sub, inputGlob)
return matches
}
- // condition to support network cve related globs
- matches, _ = fs.Glob(os.DirFS(filepath.Join(templateDir, "network")), inputGlob)
- return matches
}
var matched []string
- matches, err := fs.Glob(c.templatesFS, relPath)
- if len(matches) != 0 {
- matched = append(matched, matches...)
+ var matches []string
+ if c.templatesFS == nil {
+ var err error
+ matches, err = filepath.Glob(relPath)
+ if len(matches) != 0 {
+ matched = append(matched, matches...)
+ } else {
+ matched = append(matched, OldPathsResolver(relPath)...)
+ }
+ if err != nil && len(matched) == 0 {
+ return nil, errors.Errorf("wildcard found, but unable to glob: %s\n", err)
+ }
} else {
- matched = append(matched, OldPathsResolver(relPath)...)
- }
- if err != nil && len(matched) == 0 {
- return nil, errors.Errorf("wildcard found, but unable to glob: %s\n", err)
+ var err error
+ matches, err = fs.Glob(c.templatesFS, relPath)
+ if len(matches) != 0 {
+ matched = append(matched, matches...)
+ } else {
+ matched = append(matched, OldPathsResolver(relPath)...)
+ }
+ if err != nil && len(matched) == 0 {
+ return nil, errors.Errorf("wildcard found, but unable to glob: %s\n", err)
+ }
}
results := make([]string, 0, len(matches))
for _, match := range matches {
@@ -175,11 +215,23 @@ func (c *DiskCatalog) findGlobPathMatches(absPath string, processed map[string]s
// findFileMatches finds if a path is an absolute file. If the path
// is a file, it returns true otherwise false with no errors.
func (c *DiskCatalog) findFileMatches(absPath string, processed map[string]struct{}) (match string, matched bool, err error) {
- info, err := os.Stat(absPath)
+ if c.templatesFS != nil {
+ absPath = strings.TrimPrefix(absPath, "/")
+ }
+ var info fs.File
+ if c.templatesFS == nil {
+ info, err = os.Open(absPath)
+ } else {
+ info, err = c.templatesFS.Open(absPath)
+ }
+ if err != nil {
+ return "", false, err
+ }
+ stat, err := info.Stat()
if err != nil {
return "", false, err
}
- if !info.Mode().IsRegular() {
+ if !stat.Mode().IsRegular() {
return "", false, nil
}
if _, ok := processed[absPath]; !ok {
@@ -192,22 +244,43 @@ func (c *DiskCatalog) findFileMatches(absPath string, processed map[string]struc
// findDirectoryMatches finds matches for templates from a directory
func (c *DiskCatalog) findDirectoryMatches(absPath string, processed map[string]struct{}) ([]string, error) {
var results []string
- err := filepath.WalkDir(
- absPath,
- func(path string, d fs.DirEntry, err error) error {
- // continue on errors
- if err != nil {
+ var err error
+ if c.templatesFS == nil {
+ err = filepath.WalkDir(
+ absPath,
+ func(path string, d fs.DirEntry, err error) error {
+ // continue on errors
+ if err != nil {
+ return nil
+ }
+ if !d.IsDir() && config.GetTemplateFormatFromExt(path) != config.Unknown {
+ if _, ok := processed[path]; !ok {
+ results = append(results, path)
+ processed[path] = struct{}{}
+ }
+ }
return nil
- }
- if !d.IsDir() && config.GetTemplateFormatFromExt(path) != config.Unknown {
- if _, ok := processed[path]; !ok {
- results = append(results, path)
- processed[path] = struct{}{}
+ },
+ )
+ } else {
+ err = fs.WalkDir(
+ c.templatesFS,
+ absPath,
+ func(path string, d fs.DirEntry, err error) error {
+ // continue on errors
+ if err != nil {
+ return nil
}
- }
- return nil
- },
- )
+ if !d.IsDir() && config.GetTemplateFormatFromExt(path) != config.Unknown {
+ if _, ok := processed[path]; !ok {
+ results = append(results, path)
+ processed[path] = struct{}{}
+ }
+ }
+ return nil
+ },
+ )
+ }
return results, err
}
From 9f35b86f566e906797ed83ad3b9fdeee24881fa8 Mon Sep 17 00:00:00 2001
From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com>
Date: Mon, 24 Jun 2024 05:28:13 +0000
Subject: [PATCH 069/283] chore(deps): bump
github.com/projectdiscovery/retryablehttp-go
Bumps [github.com/projectdiscovery/retryablehttp-go](https://github.com/projectdiscovery/retryablehttp-go) from 1.0.64 to 1.0.66.
- [Release notes](https://github.com/projectdiscovery/retryablehttp-go/releases)
- [Commits](https://github.com/projectdiscovery/retryablehttp-go/compare/v1.0.64...v1.0.66)
---
updated-dependencies:
- dependency-name: github.com/projectdiscovery/retryablehttp-go
dependency-type: direct:production
update-type: version-update:semver-patch
...
Signed-off-by: dependabot[bot]
---
go.mod | 13 +++++++------
go.sum | 25 ++++++++++++++-----------
2 files changed, 21 insertions(+), 17 deletions(-)
diff --git a/go.mod b/go.mod
index 27a4f2b734..81592fed6c 100644
--- a/go.mod
+++ b/go.mod
@@ -20,12 +20,12 @@ require (
github.com/olekukonko/tablewriter v0.0.5
github.com/pkg/errors v0.9.1
github.com/projectdiscovery/clistats v0.0.20
- github.com/projectdiscovery/fastdialer v0.1.4-0.20240611222741-427bffb2ea13
- github.com/projectdiscovery/hmap v0.0.46
+ github.com/projectdiscovery/fastdialer v0.1.5
+ github.com/projectdiscovery/hmap v0.0.47
github.com/projectdiscovery/interactsh v1.2.0
github.com/projectdiscovery/rawhttp v0.1.51
- github.com/projectdiscovery/retryabledns v1.0.63
- github.com/projectdiscovery/retryablehttp-go v1.0.64
+ github.com/projectdiscovery/retryabledns v1.0.64
+ github.com/projectdiscovery/retryablehttp-go v1.0.66
github.com/projectdiscovery/yamldoc-go v1.0.4
github.com/remeh/sizedwaitgroup v1.0.0
github.com/rs/xid v1.5.0
@@ -124,7 +124,7 @@ require (
github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.14.4 // indirect
github.com/aymanbagabas/go-osc52/v2 v2.0.1 // indirect
github.com/bahlo/generic-list-go v0.2.0 // indirect
- github.com/bits-and-blooms/bitset v1.8.0 // indirect
+ github.com/bits-and-blooms/bitset v1.13.0 // indirect
github.com/bits-and-blooms/bloom/v3 v3.5.0 // indirect
github.com/buger/jsonparser v1.1.1 // indirect
github.com/bytedance/sonic v1.9.1 // indirect
@@ -146,6 +146,7 @@ require (
github.com/fatih/color v1.15.0 // indirect
github.com/free5gc/util v1.0.5-0.20230511064842-2e120956883b // indirect
github.com/gabriel-vasile/mimetype v1.4.2 // indirect
+ github.com/gaissmai/bart v0.9.5 // indirect
github.com/geoffgarside/ber v1.1.0 // indirect
github.com/gin-contrib/sse v0.1.0 // indirect
github.com/gin-gonic/gin v1.9.1 // indirect
@@ -284,7 +285,7 @@ require (
github.com/pmezard/go-difflib v1.0.0 // indirect
github.com/power-devops/perfstat v0.0.0-20210106213030-5aafc221ea8c // indirect
github.com/projectdiscovery/blackrock v0.0.1 // indirect
- github.com/projectdiscovery/networkpolicy v0.0.8
+ github.com/projectdiscovery/networkpolicy v0.0.9
github.com/rivo/uniseg v0.4.6 // indirect
github.com/saintfish/chardet v0.0.0-20230101081208-5e3ef4b5456d // indirect
github.com/tklauser/go-sysconf v0.3.12 // indirect
diff --git a/go.sum b/go.sum
index 2d9878b4ca..ad89350ce1 100644
--- a/go.sum
+++ b/go.sum
@@ -186,8 +186,9 @@ github.com/beorn7/perks v0.0.0-20180321164747-3a771d992973/go.mod h1:Dwedo/Wpr24
github.com/beorn7/perks v1.0.0/go.mod h1:KWe93zE9D1o94FZ5RNwFwVgaQK1VOXiVxmqh+CedLV8=
github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6rlkpw=
github.com/bgentry/speakeasy v0.1.0/go.mod h1:+zsyZBPWlz7T6j88CTgSN5bM796AkVf0kBD4zp0CCIs=
-github.com/bits-and-blooms/bitset v1.8.0 h1:FD+XqgOZDUxxZ8hzoBFuV9+cGWY9CslN6d5MS5JVb4c=
github.com/bits-and-blooms/bitset v1.8.0/go.mod h1:7hO7Gc7Pp1vODcmWvKMRA9BNmbv6a/7QIWpPxHddWR8=
+github.com/bits-and-blooms/bitset v1.13.0 h1:bAQ9OPNFYbGHV6Nez0tmNI0RiEu7/hxlYJRUA0wFAVE=
+github.com/bits-and-blooms/bitset v1.13.0/go.mod h1:7hO7Gc7Pp1vODcmWvKMRA9BNmbv6a/7QIWpPxHddWR8=
github.com/bits-and-blooms/bloom/v3 v3.5.0 h1:AKDvi1V3xJCmSR6QhcBfHbCN4Vf8FfxeWkMNQfmAGhY=
github.com/bits-and-blooms/bloom/v3 v3.5.0/go.mod h1:Y8vrn7nk1tPIlmLtW2ZPV+W7StdVMor6bC1xgpjMZFs=
github.com/bluele/gcache v0.0.2 h1:WcbfdXICg7G/DGBh1PFfcirkWOQV+v077yF1pSy3DGw=
@@ -326,6 +327,8 @@ github.com/fsnotify/fsnotify v1.6.0 h1:n+5WquG0fcWoWp6xPWfHdbskMCQaFnG6PfBrh1Ky4
github.com/fsnotify/fsnotify v1.6.0/go.mod h1:sl3t1tCWJFWoRz9R8WJCbQihKKwmorjAbSClcnxKAGw=
github.com/gabriel-vasile/mimetype v1.4.2 h1:w5qFW6JKBz9Y393Y4q372O9A7cUSequkh1Q7OhCmWKU=
github.com/gabriel-vasile/mimetype v1.4.2/go.mod h1:zApsH/mKG4w07erKIaJPFiX0Tsq9BFQgN3qGY5GnNgA=
+github.com/gaissmai/bart v0.9.5 h1:vy+r4Px6bjZ+v2QYXAsg63vpz9IfzdW146A8Cn4GPIo=
+github.com/gaissmai/bart v0.9.5/go.mod h1:KHeYECXQiBjTzQz/om2tqn3sZF1J7hw9m6z41ftj3fg=
github.com/geoffgarside/ber v1.1.0 h1:qTmFG4jJbwiSzSXoNJeHcOprVzZ8Ulde2Rrrifu5U9w=
github.com/geoffgarside/ber v1.1.0/go.mod h1:jVPKeCbj6MvQZhwLYsGwaGI52oUorHoHKNecGT85ZCc=
github.com/getkin/kin-openapi v0.123.0 h1:zIik0mRwFNLyvtXK274Q6ut+dPh6nlxBp0x7mNrPhs8=
@@ -833,8 +836,8 @@ github.com/projectdiscovery/clistats v0.0.20 h1:5jO5SLiRJ7f0nDV0ndBNmBeesbROouPo
github.com/projectdiscovery/clistats v0.0.20/go.mod h1:GJ2av0KnOvK0AISQnP8hyDclYIji1LVkx2l0pwnzAu4=
github.com/projectdiscovery/dsl v0.1.3 h1:Lxlgpx8lwq/CaBjVUmwF6KmkMDnYf9IVKlMsWCF72OE=
github.com/projectdiscovery/dsl v0.1.3/go.mod h1:WAuQLD4KQ/94RASotE7cvk7X6kY7JQG9Fcqc3lRQfpI=
-github.com/projectdiscovery/fastdialer v0.1.4-0.20240611222741-427bffb2ea13 h1:mAehxpgxmvFrKcQ8taA3xQskp3gg6GayAgatTAn37bg=
-github.com/projectdiscovery/fastdialer v0.1.4-0.20240611222741-427bffb2ea13/go.mod h1:PcNscfCNrCnrkmvob90yfd+rNDMh6ZOIvRIl6HFPPXE=
+github.com/projectdiscovery/fastdialer v0.1.5 h1:5rNJ8I+GVzDsUqRSVLUqGTm7LsshfNYmCh20lLn1aaI=
+github.com/projectdiscovery/fastdialer v0.1.5/go.mod h1:oVoMl9Y6da2timhszy0okyBvhbKDSk+zqFazGV4y6pU=
github.com/projectdiscovery/fasttemplate v0.0.2 h1:h2cISk5xDhlJEinlBQS6RRx0vOlOirB2y3Yu4PJzpiA=
github.com/projectdiscovery/fasttemplate v0.0.2/go.mod h1:XYWWVMxnItd+r0GbjA1GCsUopMw1/XusuQxdyAIHMCw=
github.com/projectdiscovery/freeport v0.0.5 h1:jnd3Oqsl4S8n0KuFkE5Hm8WGDP24ITBvmyw5pFTHS8Q=
@@ -849,8 +852,8 @@ github.com/projectdiscovery/gostruct v0.0.2 h1:s8gP8ApugGM4go1pA+sVlPDXaWqNP5BBD
github.com/projectdiscovery/gostruct v0.0.2/go.mod h1:H86peL4HKwMXcQQtEa6lmC8FuD9XFt6gkNR0B/Mu5PE=
github.com/projectdiscovery/gozero v0.0.2 h1:8fJeaCjxL9tpm33uG/RsCQs6HGM/NE6eA3cjkilRQ+E=
github.com/projectdiscovery/gozero v0.0.2/go.mod h1:d8bZvDWW07LWNYWrwjZ4OO1I0cpkfqaysyDfSs9ibK8=
-github.com/projectdiscovery/hmap v0.0.46 h1:1fdZ0A8Zkg30SLl0gAbEqOYb4vB3gtokbQuYI+oXUNk=
-github.com/projectdiscovery/hmap v0.0.46/go.mod h1:QleSTt2HrokGNPpymUqb7PQ/2EEHEnaU5qKrrm51Va8=
+github.com/projectdiscovery/hmap v0.0.47 h1:NdakfvlFruevnOvehdllofg/hc0CQn2WQaLHaRcsGAk=
+github.com/projectdiscovery/hmap v0.0.47/go.mod h1:/9V6EnyTY33hAy71Quox6Ggu9ZkKp36tgepBh/NzNSg=
github.com/projectdiscovery/httpx v1.6.3 h1:TSu3zEKOHGFfMsGeB0lZEFRl6Tq4nPuIgKDXi9NnYpg=
github.com/projectdiscovery/httpx v1.6.3/go.mod h1:9FmsmkT71aTa4WnOyBIPzB8I9c0QAJ9PSVUqs2NkHRQ=
github.com/projectdiscovery/interactsh v1.2.0 h1:Al6jHiR+Usl9egYJDLJaWNHOcH8Rugk8gWMasc8Cmw8=
@@ -863,18 +866,18 @@ github.com/projectdiscovery/mapcidr v1.1.34 h1:udr83vQ7oz3kEOwlsU6NC6o08leJzSDQt
github.com/projectdiscovery/mapcidr v1.1.34/go.mod h1:1+1R6OkKSAKtWDXE9RvxXtXPoajXTYX0eiEdkqlhQqQ=
github.com/projectdiscovery/n3iwf v0.0.0-20230523120440-b8cd232ff1f5 h1:L/e8z8yw1pfT6bg35NiN7yd1XKtJap5Nk6lMwQ0RNi8=
github.com/projectdiscovery/n3iwf v0.0.0-20230523120440-b8cd232ff1f5/go.mod h1:pGW2ncnTxTxHtP9wzcIJAB+3/NMp6IiuQWd2NK7K+oc=
-github.com/projectdiscovery/networkpolicy v0.0.8 h1:XvfBaBwSDNTesSfNQP9VLk3HX9I7x7gHm028TJ5XwI8=
-github.com/projectdiscovery/networkpolicy v0.0.8/go.mod h1:xnjNqhemxUPxU+UD5Jgsc3+K8IVmcqT1SJeo6UzMtkI=
+github.com/projectdiscovery/networkpolicy v0.0.9 h1:IrlDoYZagNNO8y+7iZeHT8k5izE+nek7TdtvEBwCxqk=
+github.com/projectdiscovery/networkpolicy v0.0.9/go.mod h1:XFJ2Lnv8BE/ziQCFjBHMsH1w6VmkPiQtk+NlBpdMU7M=
github.com/projectdiscovery/ratelimit v0.0.45 h1:h28oF+hJ0CHcdBZozT1Go7ppWmzTxSXDKNNh2G1Ot9Q=
github.com/projectdiscovery/ratelimit v0.0.45/go.mod h1:1vSJUseDS7SjNwIBi9wNRcgsMKNTLxy/GfdlLFVbgI4=
github.com/projectdiscovery/rawhttp v0.1.51 h1:T4sskU2fsX9zNR7nyae0tD/0fliIs73KwBzyoJwBs10=
github.com/projectdiscovery/rawhttp v0.1.51/go.mod h1:ejBaw3dCfJSfbtFv0ib6YfFSjQzt7ZdRVpQWJASxIUs=
github.com/projectdiscovery/rdap v0.9.1-0.20221108103045-9865884d1917 h1:m03X4gBVSorSzvmm0bFa7gDV4QNSOWPL/fgZ4kTXBxk=
github.com/projectdiscovery/rdap v0.9.1-0.20221108103045-9865884d1917/go.mod h1:JxXtZC9e195awe7EynrcnBJmFoad/BNDzW9mzFkK8Sg=
-github.com/projectdiscovery/retryabledns v1.0.63 h1:Ijt47ybwf+iIwul606NlPKPN+ouhOYPeA9sLHVgqLG4=
-github.com/projectdiscovery/retryabledns v1.0.63/go.mod h1:lTs48OYJnMFuuBzT+3z3PrZ58K0OUBgP7Y4o3ttBwb0=
-github.com/projectdiscovery/retryablehttp-go v1.0.64 h1:90VkUuWKm/JW5Tym0u+N1qseKe4gC5Rp048VJBLcxv8=
-github.com/projectdiscovery/retryablehttp-go v1.0.64/go.mod h1:fXVLBcwKwBrCVEgi3FrbLPtSTIczyJEKIlp1WD6VTI0=
+github.com/projectdiscovery/retryabledns v1.0.64 h1:bhaKarpUnPjTXN9A0ApA/IA4SWfugdCke1yT0YxIj0k=
+github.com/projectdiscovery/retryabledns v1.0.64/go.mod h1:n1N7LMgxMNQT/bUoE5/OQZBTxVApkfeAvDtYQ1OnuAg=
+github.com/projectdiscovery/retryablehttp-go v1.0.66 h1:dY+sGVf/O8CvKcSdTx0rxJoKG+HyOScz5T89TijZohU=
+github.com/projectdiscovery/retryablehttp-go v1.0.66/go.mod h1:3QJt06j4hEZVpJOr31u49I3mDnqbAzhcjJX8Yvn/xnY=
github.com/projectdiscovery/sarif v0.0.1 h1:C2Tyj0SGOKbCLgHrx83vaE6YkzXEVrMXYRGLkKCr/us=
github.com/projectdiscovery/sarif v0.0.1/go.mod h1:cEYlDu8amcPf6b9dSakcz2nNnJsoz4aR6peERwV+wuQ=
github.com/projectdiscovery/stringsutil v0.0.2 h1:uzmw3IVLJSMW1kEg8eCStG/cGbYYZAja8BH3LqqJXMA=
From 61029d05c3e0e0fa77c4a9be120d86c3f98d0528 Mon Sep 17 00:00:00 2001
From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com>
Date: Mon, 24 Jun 2024 05:56:07 +0000
Subject: [PATCH 070/283] chore(deps): bump
github.com/projectdiscovery/retryabledns
Bumps [github.com/projectdiscovery/retryabledns](https://github.com/projectdiscovery/retryabledns) from 1.0.63 to 1.0.65.
- [Release notes](https://github.com/projectdiscovery/retryabledns/releases)
- [Commits](https://github.com/projectdiscovery/retryabledns/compare/v1.0.63...v1.0.65)
---
updated-dependencies:
- dependency-name: github.com/projectdiscovery/retryabledns
dependency-type: direct:production
update-type: version-update:semver-patch
...
Signed-off-by: dependabot[bot]
---
go.mod | 2 +-
go.sum | 4 ++--
2 files changed, 3 insertions(+), 3 deletions(-)
diff --git a/go.mod b/go.mod
index 81592fed6c..9b4c511c7c 100644
--- a/go.mod
+++ b/go.mod
@@ -24,7 +24,7 @@ require (
github.com/projectdiscovery/hmap v0.0.47
github.com/projectdiscovery/interactsh v1.2.0
github.com/projectdiscovery/rawhttp v0.1.51
- github.com/projectdiscovery/retryabledns v1.0.64
+ github.com/projectdiscovery/retryabledns v1.0.65
github.com/projectdiscovery/retryablehttp-go v1.0.66
github.com/projectdiscovery/yamldoc-go v1.0.4
github.com/remeh/sizedwaitgroup v1.0.0
diff --git a/go.sum b/go.sum
index ad89350ce1..e815c24d5b 100644
--- a/go.sum
+++ b/go.sum
@@ -874,8 +874,8 @@ github.com/projectdiscovery/rawhttp v0.1.51 h1:T4sskU2fsX9zNR7nyae0tD/0fliIs73Kw
github.com/projectdiscovery/rawhttp v0.1.51/go.mod h1:ejBaw3dCfJSfbtFv0ib6YfFSjQzt7ZdRVpQWJASxIUs=
github.com/projectdiscovery/rdap v0.9.1-0.20221108103045-9865884d1917 h1:m03X4gBVSorSzvmm0bFa7gDV4QNSOWPL/fgZ4kTXBxk=
github.com/projectdiscovery/rdap v0.9.1-0.20221108103045-9865884d1917/go.mod h1:JxXtZC9e195awe7EynrcnBJmFoad/BNDzW9mzFkK8Sg=
-github.com/projectdiscovery/retryabledns v1.0.64 h1:bhaKarpUnPjTXN9A0ApA/IA4SWfugdCke1yT0YxIj0k=
-github.com/projectdiscovery/retryabledns v1.0.64/go.mod h1:n1N7LMgxMNQT/bUoE5/OQZBTxVApkfeAvDtYQ1OnuAg=
+github.com/projectdiscovery/retryabledns v1.0.65 h1:wOtMuVkrcrGkzaT8Jixb3BtUcJODXLRz+z9EjftmFsQ=
+github.com/projectdiscovery/retryabledns v1.0.65/go.mod h1:lf3xRdFhGh55fWVlzK1oAaDPrsKliqH3gGvsFNAOkCk=
github.com/projectdiscovery/retryablehttp-go v1.0.66 h1:dY+sGVf/O8CvKcSdTx0rxJoKG+HyOScz5T89TijZohU=
github.com/projectdiscovery/retryablehttp-go v1.0.66/go.mod h1:3QJt06j4hEZVpJOr31u49I3mDnqbAzhcjJX8Yvn/xnY=
github.com/projectdiscovery/sarif v0.0.1 h1:C2Tyj0SGOKbCLgHrx83vaE6YkzXEVrMXYRGLkKCr/us=
From 91d3b668fdcc2e368896e9dc7eb9a0f8b3ab5679 Mon Sep 17 00:00:00 2001
From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com>
Date: Mon, 24 Jun 2024 22:09:02 +0000
Subject: [PATCH 071/283] chore(deps): bump
github.com/hashicorp/go-retryablehttp
Bumps [github.com/hashicorp/go-retryablehttp](https://github.com/hashicorp/go-retryablehttp) from 0.7.2 to 0.7.7.
- [Changelog](https://github.com/hashicorp/go-retryablehttp/blob/main/CHANGELOG.md)
- [Commits](https://github.com/hashicorp/go-retryablehttp/compare/v0.7.2...v0.7.7)
---
updated-dependencies:
- dependency-name: github.com/hashicorp/go-retryablehttp
dependency-type: indirect
...
Signed-off-by: dependabot[bot]
---
go.mod | 4 ++--
go.sum | 12 ++++++------
2 files changed, 8 insertions(+), 8 deletions(-)
diff --git a/go.mod b/go.mod
index 9b4c511c7c..be2c0ca475 100644
--- a/go.mod
+++ b/go.mod
@@ -143,7 +143,7 @@ require (
github.com/docker/cli v24.0.5+incompatible // indirect
github.com/docker/docker v24.0.9+incompatible // indirect
github.com/docker/go-connections v0.4.0 // indirect
- github.com/fatih/color v1.15.0 // indirect
+ github.com/fatih/color v1.16.0 // indirect
github.com/free5gc/util v1.0.5-0.20230511064842-2e120956883b // indirect
github.com/gabriel-vasile/mimetype v1.4.2 // indirect
github.com/gaissmai/bart v0.9.5 // indirect
@@ -268,7 +268,7 @@ require (
github.com/google/uuid v1.6.0 // indirect
github.com/gorilla/css v1.0.1 // indirect
github.com/hashicorp/go-cleanhttp v0.5.2 // indirect
- github.com/hashicorp/go-retryablehttp v0.7.2 // indirect
+ github.com/hashicorp/go-retryablehttp v0.7.7 // indirect
github.com/hdm/jarm-go v0.0.7 // indirect
github.com/itchyny/timefmt-go v0.1.5 // indirect
github.com/klauspost/cpuid/v2 v2.2.5 // indirect
diff --git a/go.sum b/go.sum
index e815c24d5b..5cda424843 100644
--- a/go.sum
+++ b/go.sum
@@ -310,8 +310,8 @@ github.com/envoyproxy/go-control-plane v0.9.1-0.20191026205805-5f8ba28d4473/go.m
github.com/envoyproxy/go-control-plane v0.9.4/go.mod h1:6rpuAdCZL397s3pYoYcLgu1mIlRU8Am5FuJP05cCM98=
github.com/envoyproxy/protoc-gen-validate v0.1.0/go.mod h1:iSmxcyjqTsJpI2R4NaDN7+kN2VEUnK/pcBlmesArF7c=
github.com/fatih/color v1.7.0/go.mod h1:Zm6kSWBoL9eyXnKyktHP6abPY2pDugNf5KwzbycvMj4=
-github.com/fatih/color v1.15.0 h1:kOqh6YHBtK8aywxGerMG2Eq3H6Qgoqeo13Bk2Mv/nBs=
-github.com/fatih/color v1.15.0/go.mod h1:0h5ZqXfHYED7Bhv2ZJamyIOUej9KtShiJESRwBDUSsw=
+github.com/fatih/color v1.16.0 h1:zmkK9Ngbjj+K0yRhTVONQh1p/HknKYSlNT+vZCzyokM=
+github.com/fatih/color v1.16.0/go.mod h1:fL2Sau1YI5c0pdGEVCbKQbLXB6edEj1ZgiY4NijnWvE=
github.com/fatih/structs v1.1.0 h1:Q7juDM0QtcnhCpeyLGQKyg4TOIghuNXrkL32pHAUMxo=
github.com/fatih/structs v1.1.0/go.mod h1:9NiDSp5zOcgEDl+j00MP/WkGVPOlPRLejGD8Ga6PJ7M=
github.com/fortytw2/leaktest v1.3.0 h1:u8491cBMTQ8ft8aeV+adlcytMZylmA5nnwwkRZjI8vw=
@@ -531,13 +531,13 @@ github.com/hashicorp/errwrap v1.0.0/go.mod h1:YH+1FKiLXxHSkmPseP+kNlulaMuP3n2brv
github.com/hashicorp/go-cleanhttp v0.5.1/go.mod h1:JpRdi6/HCYpAwUzNwuwqhbovhLtngrth3wmdIIUrZ80=
github.com/hashicorp/go-cleanhttp v0.5.2 h1:035FKYIWjmULyFRBKPs8TBQoi0x6d9G4xc9neXJWAZQ=
github.com/hashicorp/go-cleanhttp v0.5.2/go.mod h1:kO/YDlP8L1346E6Sodw+PrpBSV4/SoxCXGY6BqNFT48=
-github.com/hashicorp/go-hclog v0.9.2 h1:CG6TE5H9/JXsFWJCfoIVpKFIkFe6ysEuHirp4DxCsHI=
-github.com/hashicorp/go-hclog v0.9.2/go.mod h1:5CU+agLiy3J7N7QjHK5d05KxGsuXiQLrjA0H7acj2lQ=
+github.com/hashicorp/go-hclog v1.6.3 h1:Qr2kF+eVWjTiYmU7Y31tYlP1h0q/X3Nl3tPGdaB11/k=
+github.com/hashicorp/go-hclog v1.6.3/go.mod h1:W4Qnvbt70Wk/zYJryRzDRU/4r0kIg0PVHBcfoyhpF5M=
github.com/hashicorp/go-immutable-radix v1.0.0/go.mod h1:0y9vanUI8NX6FsYoO3zeMjhV/C5i9g4Q3DwcSNZ4P60=
github.com/hashicorp/go-msgpack v0.5.3/go.mod h1:ahLV/dePpqEmjfWmKiqvPkv/twdG7iPBM1vqhUKIvfM=
github.com/hashicorp/go-multierror v1.0.0/go.mod h1:dHtQlpGsu+cZNNAkkCN/P3hoUDHhCYQXV3UM06sGGrk=
-github.com/hashicorp/go-retryablehttp v0.7.2 h1:AcYqCvkpalPnPF2pn0KamgwamS42TqUDDYFRKq/RAd0=
-github.com/hashicorp/go-retryablehttp v0.7.2/go.mod h1:Jy/gPYAdjqffZ/yFGCFV2doI5wjtH1ewM9u8iYVjtX8=
+github.com/hashicorp/go-retryablehttp v0.7.7 h1:C8hUCYzor8PIfXHa4UrZkU4VvK8o9ISHxT2Q8+VepXU=
+github.com/hashicorp/go-retryablehttp v0.7.7/go.mod h1:pkQpWZeYWskR+D1tR2O5OcBFOxfA7DoAO6xtkuQnHTk=
github.com/hashicorp/go-rootcerts v1.0.0/go.mod h1:K6zTfqpRlCUIjkwsN4Z+hiSfzSTQa6eBIzfwKfwNnHU=
github.com/hashicorp/go-sockaddr v1.0.0/go.mod h1:7Xibr9yA9JjQq1JpNB2Vw7kxv8xerXegt+ozgdvDeDU=
github.com/hashicorp/go-syslog v1.0.0/go.mod h1:qPfqrKkXGihmCqbJM2mZgkZGvKG1dFdvsLplgctolz4=
From c7006a9168c116354cc5b729eb17b515bb47428f Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Do=C4=9Fan=20Can=20Bak=C4=B1r?=
Date: Tue, 25 Jun 2024 12:26:18 +0300
Subject: [PATCH 072/283] use `stringsutil.ContainsAll`
---
go.mod | 6 +++---
go.sum | 12 ++++++------
pkg/input/provider/interface.go | 3 ++-
.../common/hosterrorscache/hosterrorscache.go | 3 ++-
pkg/protocols/http/httpclientpool/options.go | 3 ++-
pkg/templates/preprocessors.go | 3 ++-
6 files changed, 17 insertions(+), 13 deletions(-)
diff --git a/go.mod b/go.mod
index be2c0ca475..ae7985a9df 100644
--- a/go.mod
+++ b/go.mod
@@ -21,7 +21,7 @@ require (
github.com/pkg/errors v0.9.1
github.com/projectdiscovery/clistats v0.0.20
github.com/projectdiscovery/fastdialer v0.1.5
- github.com/projectdiscovery/hmap v0.0.47
+ github.com/projectdiscovery/hmap v0.0.48
github.com/projectdiscovery/interactsh v1.2.0
github.com/projectdiscovery/rawhttp v0.1.51
github.com/projectdiscovery/retryabledns v1.0.65
@@ -82,7 +82,7 @@ require (
github.com/projectdiscovery/fasttemplate v0.0.2
github.com/projectdiscovery/go-smb2 v0.0.0-20240129202741-052cc450c6cb
github.com/projectdiscovery/goflags v0.1.56
- github.com/projectdiscovery/gologger v1.1.12
+ github.com/projectdiscovery/gologger v1.1.13
github.com/projectdiscovery/gostruct v0.0.2
github.com/projectdiscovery/gozero v0.0.2
github.com/projectdiscovery/httpx v1.6.3
@@ -94,7 +94,7 @@ require (
github.com/projectdiscovery/tlsx v1.1.6
github.com/projectdiscovery/uncover v1.0.8
github.com/projectdiscovery/useragent v0.0.56
- github.com/projectdiscovery/utils v0.1.4
+ github.com/projectdiscovery/utils v0.1.6
github.com/projectdiscovery/wappalyzergo v0.1.6
github.com/redis/go-redis/v9 v9.1.0
github.com/seh-msft/burpxml v1.0.1
diff --git a/go.sum b/go.sum
index 5cda424843..ab76fb2918 100644
--- a/go.sum
+++ b/go.sum
@@ -846,14 +846,14 @@ github.com/projectdiscovery/go-smb2 v0.0.0-20240129202741-052cc450c6cb h1:rutG90
github.com/projectdiscovery/go-smb2 v0.0.0-20240129202741-052cc450c6cb/go.mod h1:FLjF1DmZ+POoGEiIQdWuYVwS++C/GwpX8YaCsTSm1RY=
github.com/projectdiscovery/goflags v0.1.56 h1:tJYiZN7s9Jk9DxfYOUiqOoybaIDlXyX4ZgT4B/06SyU=
github.com/projectdiscovery/goflags v0.1.56/go.mod h1:DsGF0NPpM5hGg75N3MTSvWJ4MIT7HFEAOEeWZ074+Fg=
-github.com/projectdiscovery/gologger v1.1.12 h1:uX/QkQdip4PubJjjG0+uk5DtyAi1ANPJUvpmimXqv4A=
-github.com/projectdiscovery/gologger v1.1.12/go.mod h1:DI8nywPLERS5mo8QEA9E7gd5HZ3Je14SjJBH3F5/kLw=
+github.com/projectdiscovery/gologger v1.1.13 h1:7h4yCnMk+DQ/43v+A3NuA1uuI4XlJd0W0xvTZioD05E=
+github.com/projectdiscovery/gologger v1.1.13/go.mod h1:PAsVPr2aHoopCzE0BOnp2bPCv+BU/Oq81ygcP3xVwgI=
github.com/projectdiscovery/gostruct v0.0.2 h1:s8gP8ApugGM4go1pA+sVlPDXaWqNP5BBDDSv7VEdG1M=
github.com/projectdiscovery/gostruct v0.0.2/go.mod h1:H86peL4HKwMXcQQtEa6lmC8FuD9XFt6gkNR0B/Mu5PE=
github.com/projectdiscovery/gozero v0.0.2 h1:8fJeaCjxL9tpm33uG/RsCQs6HGM/NE6eA3cjkilRQ+E=
github.com/projectdiscovery/gozero v0.0.2/go.mod h1:d8bZvDWW07LWNYWrwjZ4OO1I0cpkfqaysyDfSs9ibK8=
-github.com/projectdiscovery/hmap v0.0.47 h1:NdakfvlFruevnOvehdllofg/hc0CQn2WQaLHaRcsGAk=
-github.com/projectdiscovery/hmap v0.0.47/go.mod h1:/9V6EnyTY33hAy71Quox6Ggu9ZkKp36tgepBh/NzNSg=
+github.com/projectdiscovery/hmap v0.0.48 h1:pbxoDcwZWSdaqZ3QgaG5Aw1G1qEFN2zeQco6eWeROAo=
+github.com/projectdiscovery/hmap v0.0.48/go.mod h1:KMSbPJoIgI+9PkG6alZ0INEMRiIGnND128RUjxiY+8I=
github.com/projectdiscovery/httpx v1.6.3 h1:TSu3zEKOHGFfMsGeB0lZEFRl6Tq4nPuIgKDXi9NnYpg=
github.com/projectdiscovery/httpx v1.6.3/go.mod h1:9FmsmkT71aTa4WnOyBIPzB8I9c0QAJ9PSVUqs2NkHRQ=
github.com/projectdiscovery/interactsh v1.2.0 h1:Al6jHiR+Usl9egYJDLJaWNHOcH8Rugk8gWMasc8Cmw8=
@@ -888,8 +888,8 @@ github.com/projectdiscovery/uncover v1.0.8 h1:nE8bYJuwhqk0BEMRQRhNbjPGMy40A9gkiM
github.com/projectdiscovery/uncover v1.0.8/go.mod h1:1SwsNqjwMkJEzJQ7lQr5AHIdYd+BQlyqhO5IwIYmIAM=
github.com/projectdiscovery/useragent v0.0.56 h1:KpUtOgq1vUerPrgRFZAIr1hF0lDxA2RF6YkT2PLwSoQ=
github.com/projectdiscovery/useragent v0.0.56/go.mod h1:LAh9vudxXPA5JfNN3+1g/Unqvulq0BDkjxK7IrE+e/s=
-github.com/projectdiscovery/utils v0.1.4 h1:VSULN/RBEXbkgWuhrDey84ZDnSXtYMSev9p+QHXXDeI=
-github.com/projectdiscovery/utils v0.1.4/go.mod h1:mXs6OOeG9l/dVchjB2PGvQO3+wuMiE14Y/kmHeKogoM=
+github.com/projectdiscovery/utils v0.1.6 h1:fzdN+oCTics3hPPOEV1FWqf/Hpenav5Ml/73aEwpTZc=
+github.com/projectdiscovery/utils v0.1.6/go.mod h1:kOnMoFfuQMY8U+nh8e32odjC00iEbiLVBcDxyAZFwCc=
github.com/projectdiscovery/wappalyzergo v0.1.6 h1:4iZDEnytsnAqjQgD5K6lmAzVx++KSQeMCDfomDtQM1M=
github.com/projectdiscovery/wappalyzergo v0.1.6/go.mod h1:wBYGKmA5BQp/NWsAy1q/jSH8N1LHWQ/LV26DuR+KzPM=
github.com/projectdiscovery/yamldoc-go v1.0.4 h1:eZoESapnMw6WAHiVgRwNqvbJEfNHEH148uthhFbG5jE=
diff --git a/pkg/input/provider/interface.go b/pkg/input/provider/interface.go
index cb48fc7676..b7ad4b4480 100644
--- a/pkg/input/provider/interface.go
+++ b/pkg/input/provider/interface.go
@@ -14,6 +14,7 @@ import (
"github.com/projectdiscovery/nuclei/v3/pkg/protocols/common/generators"
configTypes "github.com/projectdiscovery/nuclei/v3/pkg/types"
errorutil "github.com/projectdiscovery/utils/errors"
+ stringsutil "github.com/projectdiscovery/utils/strings"
)
var (
@@ -32,7 +33,7 @@ func IsErrNotImplemented(err error) bool {
if err == nil {
return false
}
- if strings.Contains(err.Error(), "provider") && strings.Contains(err.Error(), "does not implement") {
+ if stringsutil.ContainsAll(err.Error(), "provider", "does not implement") {
return true
}
return false
diff --git a/pkg/protocols/common/hosterrorscache/hosterrorscache.go b/pkg/protocols/common/hosterrorscache/hosterrorscache.go
index 38a7ff3290..1630e97c7d 100644
--- a/pkg/protocols/common/hosterrorscache/hosterrorscache.go
+++ b/pkg/protocols/common/hosterrorscache/hosterrorscache.go
@@ -14,6 +14,7 @@ import (
"github.com/projectdiscovery/nuclei/v3/pkg/protocols/common/contextargs"
"github.com/projectdiscovery/nuclei/v3/pkg/types/nucleierr"
"github.com/projectdiscovery/utils/errkit"
+ stringsutil "github.com/projectdiscovery/utils/strings"
)
// CacheInterface defines the signature of the hosterrorscache so that
@@ -200,7 +201,7 @@ func (c *Cache) checkError(err error) bool {
errX := errkit.FromError(err)
tmp := errX.Cause()
cause := tmp.Error()
- if strings.Contains(cause, "ReadStatusLine:") && strings.Contains(cause, "read: connection reset by peer") {
+ if stringsutil.ContainsAll(cause, "ReadStatusLine:", "read: connection reset by peer") {
// this is a FP and should not be counted as a host error
// because server closes connection when it reads corrupted bytes which we send via rawhttp
return false
diff --git a/pkg/protocols/http/httpclientpool/options.go b/pkg/protocols/http/httpclientpool/options.go
index f24e421b86..92e42fe96b 100644
--- a/pkg/protocols/http/httpclientpool/options.go
+++ b/pkg/protocols/http/httpclientpool/options.go
@@ -7,6 +7,7 @@ import (
"time"
"github.com/projectdiscovery/rawhttp"
+ stringsutil "github.com/projectdiscovery/utils/strings"
urlutil "github.com/projectdiscovery/utils/url"
)
@@ -36,7 +37,7 @@ func SendRawRequest(client *rawhttp.Client, opts *RawHttpRequestOpts) (*http.Res
resp, err := client.DoRawWithOptions(opts.Method, opts.URL, opts.Path, opts.Headers, opts.Body, opts.Options)
if err != nil {
cause := err.Error()
- if strings.Contains(cause, "ReadStatusLine: ") && strings.Contains(cause, "read: connection reset by peer") {
+ if stringsutil.ContainsAll(cause, "ReadStatusLine: ", "read: connection reset by peer") {
// this error is caused when rawhttp client sends a corrupted or malformed request packet to server
// some servers may attempt gracefully shutdown but most will just abruptly close the connection which results
// in a connection reset by peer error and this can be safely assumed as 400 Bad Request in terms of normal http flow
diff --git a/pkg/templates/preprocessors.go b/pkg/templates/preprocessors.go
index 054a0bc41e..7cf6f8e21a 100644
--- a/pkg/templates/preprocessors.go
+++ b/pkg/templates/preprocessors.go
@@ -5,6 +5,7 @@ import (
"regexp"
"strings"
+ stringsutil "github.com/projectdiscovery/utils/strings"
"github.com/segmentio/ksuid"
)
@@ -48,7 +49,7 @@ func (r *randStrPreprocessor) ProcessNReturnData(data []byte) ([]byte, map[strin
continue
}
value := expression[1]
- if strings.Contains(value, "(") || strings.Contains(value, ")") {
+ if stringsutil.ContainsAny(value, "(", ")") {
continue
}
From 381ebba6a2f56dfc99cf92312728ab51c099449b Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Kristinn=20Vikar=20J=C3=B3nsson?=
<93918469+KristinnVikar@users.noreply.github.com>
Date: Thu, 27 Jun 2024 07:44:43 +0000
Subject: [PATCH 073/283] Clustering performance improvements (#5319)
* Clustering performance improvements
* IsClusterable filters out beforehand, update test to mirror that
* inverse IsClusterable
This makes much more sense
* HashMap based clustering
* furthur improvements to clustering
---------
Co-authored-by: Tarun Koyalwar
---
pkg/protocols/dns/cluster.go | 41 +++++------
pkg/protocols/http/cluster.go | 39 ++++------
pkg/protocols/http/cluster_test.go | 7 +-
pkg/protocols/ssl/ssl.go | 20 +++---
pkg/templates/cluster.go | 110 +++++++++++------------------
pkg/utils/utils.go | 14 ++++
6 files changed, 107 insertions(+), 124 deletions(-)
diff --git a/pkg/protocols/dns/cluster.go b/pkg/protocols/dns/cluster.go
index f2f5734785..86852b4d78 100644
--- a/pkg/protocols/dns/cluster.go
+++ b/pkg/protocols/dns/cluster.go
@@ -1,27 +1,24 @@
package dns
-// CanCluster returns true if the request can be clustered.
-//
-// This used by the clustering engine to decide whether two requests
-// are similar enough to be considered one and can be checked by
-// just adding the matcher/extractors for the request and the correct IDs.
-func (request *Request) CanCluster(other *Request) bool {
- if len(request.Resolvers) > 0 || request.Trace || request.ID != "" {
- return false
- }
- if request.Name != other.Name ||
- request.class != other.class ||
- request.Retries != other.Retries ||
- request.question != other.question {
- return false
- }
+import (
+ "fmt"
+
+ "github.com/cespare/xxhash"
+)
+
+
+// TmplClusterKey generates a unique key for the request
+// to be used in the clustering process.
+func (request *Request) TmplClusterKey() uint64 {
+ recursion := ""
if request.Recursion != nil {
- if other.Recursion == nil {
- return false
- }
- if *request.Recursion != *other.Recursion {
- return false
- }
+ recursion = fmt.Sprintf("%t", *request.Recursion)
}
- return true
+ inp := fmt.Sprintf("%s-%d-%d-%d-%s", request.Name, request.class, request.Retries, request.question, recursion)
+ return xxhash.Sum64String(inp)
+}
+
+// IsClusterable returns true if the request is eligible to be clustered.
+func (request *Request) IsClusterable() bool {
+ return !(len(request.Resolvers) > 0 || request.Trace || request.ID != "")
}
diff --git a/pkg/protocols/http/cluster.go b/pkg/protocols/http/cluster.go
index 3839817454..aa95c32bae 100644
--- a/pkg/protocols/http/cluster.go
+++ b/pkg/protocols/http/cluster.go
@@ -1,30 +1,21 @@
package http
import (
- sliceutil "github.com/projectdiscovery/utils/slice"
- "golang.org/x/exp/maps"
+ "fmt"
+ "strings"
+
+ "github.com/cespare/xxhash"
+ "github.com/projectdiscovery/nuclei/v3/pkg/utils"
)
-// CanCluster returns true if the request can be clustered.
-//
-// This used by the clustering engine to decide whether two requests
-// are similar enough to be considered one and can be checked by
-// just adding the matcher/extractors for the request and the correct IDs.
-func (request *Request) CanCluster(other *Request) bool {
- if len(request.Payloads) > 0 || len(request.Fuzzing) > 0 || len(request.Raw) > 0 || len(request.Body) > 0 || request.Unsafe || request.NeedsRequestCondition() || request.Name != "" {
- return false
- }
- if request.Method != other.Method ||
- request.MaxRedirects != other.MaxRedirects ||
- request.DisableCookie != other.DisableCookie ||
- request.Redirects != other.Redirects {
- return false
- }
- if !sliceutil.Equal(request.Path, other.Path) {
- return false
- }
- if !maps.Equal(request.Headers, other.Headers) {
- return false
- }
- return true
+// TmplClusterKey generates a unique key for the request
+// to be used in the clustering process.
+func (request *Request) TmplClusterKey() uint64 {
+ inp := fmt.Sprintf("%s-%d-%t-%t-%s-%d", request.Method.String(), request.MaxRedirects, request.DisableCookie, request.Redirects, strings.Join(request.Path, "-"), utils.MapHash(request.Headers))
+ return xxhash.Sum64String(inp)
+}
+
+// IsClusterable returns true if the request is eligible to be clustered.
+func (request *Request) IsClusterable() bool {
+ return !(len(request.Payloads) > 0 || len(request.Fuzzing) > 0 || len(request.Raw) > 0 || len(request.Body) > 0 || request.Unsafe || request.NeedsRequestCondition() || request.Name != "")
}
diff --git a/pkg/protocols/http/cluster_test.go b/pkg/protocols/http/cluster_test.go
index a16db41ee0..da9c1e5a76 100644
--- a/pkg/protocols/http/cluster_test.go
+++ b/pkg/protocols/http/cluster_test.go
@@ -8,8 +8,11 @@ import (
func TestCanCluster(t *testing.T) {
req := &Request{Unsafe: true}
- require.False(t, req.CanCluster(&Request{}), "could cluster unsafe request")
+ require.False(t, req.IsClusterable(), "could cluster unsafe request")
req = &Request{Path: []string{"{{BaseURL}}"}, Method: HTTPMethodTypeHolder{MethodType: HTTPGet}}
- require.True(t, req.CanCluster(&Request{Path: []string{"{{BaseURL}}"}, Method: HTTPMethodTypeHolder{MethodType: HTTPGet}}), "could not cluster GET request")
+ newReq := &Request{Path: []string{"{{BaseURL}}"}, Method: HTTPMethodTypeHolder{MethodType: HTTPGet}}
+ require.True(t, req.IsClusterable(), "could not cluster GET request")
+ require.True(t, req.IsClusterable(), "could not cluster GET request")
+ require.Equal(t, req.TmplClusterKey(), newReq.TmplClusterKey(), "cluster keys should be equal")
}
diff --git a/pkg/protocols/ssl/ssl.go b/pkg/protocols/ssl/ssl.go
index 49d612b882..1abde973aa 100644
--- a/pkg/protocols/ssl/ssl.go
+++ b/pkg/protocols/ssl/ssl.go
@@ -3,8 +3,10 @@ package ssl
import (
"fmt"
"net"
+ "strings"
"time"
+ "github.com/cespare/xxhash"
"github.com/fatih/structs"
jsoniter "github.com/json-iterator/go"
"github.com/pkg/errors"
@@ -99,15 +101,15 @@ type Request struct {
options *protocols.ExecutorOptions
}
-// CanCluster returns true if the request can be clustered.
-func (request *Request) CanCluster(other *Request) bool {
- if len(request.CipherSuites) > 0 || request.MinVersion != "" || request.MaxVersion != "" {
- return false
- }
- if request.Address != other.Address || request.ScanMode != other.ScanMode {
- return false
- }
- return true
+// TmplClusterKey generates a unique key for the request
+// to be used in the clustering process.
+func (request *Request) TmplClusterKey() uint64 {
+ inp := fmt.Sprintf("%s-%s-%t-%t-%s", request.Address, request.ScanMode, request.TLSCiphersEnum, request.TLSVersionsEnum, strings.Join(request.TLSCipherTypes, ","))
+ return xxhash.Sum64String(inp)
+}
+
+func (request *Request) IsClusterable() bool {
+ return !(len(request.CipherSuites) > 0 || request.MinVersion != "" || request.MaxVersion != "")
}
// Compile compiles the request generators preparing any requests possible.
diff --git a/pkg/templates/cluster.go b/pkg/templates/cluster.go
index 603378e986..a4a4ac4bad 100644
--- a/pkg/templates/cluster.go
+++ b/pkg/templates/cluster.go
@@ -14,7 +14,6 @@ import (
"github.com/projectdiscovery/nuclei/v3/pkg/scan"
"github.com/projectdiscovery/nuclei/v3/pkg/templates/types"
cryptoutil "github.com/projectdiscovery/utils/crypto"
- mapsutil "github.com/projectdiscovery/utils/maps"
)
// Cluster clusters a list of templates into a lesser number if possible based
@@ -42,91 +41,68 @@ import (
// Finally, the engine creates a single executer with a clusteredexecuter for all templates
// in a cluster.
func Cluster(list []*Template) [][]*Template {
+ http := make(map[uint64][]*Template)
+ dns := make(map[uint64][]*Template)
+ ssl := make(map[uint64][]*Template)
+
final := [][]*Template{}
- skip := mapsutil.NewSyncLockMap[string, struct{}]()
+ // Split up templates that might be clusterable
for _, template := range list {
- key := template.Path
-
- if skip.Has(key) {
- continue
- }
-
- // We only cluster http, dns and ssl requests as of now.
- // Take care of requests that can't be clustered first.
- if len(template.RequestsHTTP) == 0 && len(template.RequestsDNS) == 0 && len(template.RequestsSSL) == 0 {
- _ = skip.Set(key, struct{}{})
- final = append(final, []*Template{template})
- continue
- }
-
// it is not possible to cluster flow and multiprotocol due to dependent execution
if template.Flow != "" || template.Options.IsMultiProtocol {
- _ = skip.Set(key, struct{}{})
final = append(final, []*Template{template})
continue
}
- _ = skip.Set(key, struct{}{})
-
- var templateType types.ProtocolType
switch {
case len(template.RequestsDNS) == 1:
- templateType = types.DNSProtocol
- case len(template.RequestsHTTP) == 1:
- templateType = types.HTTPProtocol
- case len(template.RequestsSSL) == 1:
- templateType = types.SSLProtocol
- }
-
- // Find any/all similar matching request that is identical to
- // this one and cluster them together for http protocol only.
- cluster := []*Template{}
- for _, other := range list {
- otherKey := other.Path
-
- if skip.Has(otherKey) {
- continue
- }
-
- // it is not possible to cluster flow and multiprotocol due to dependent execution
- if other.Flow != "" || other.Options.IsMultiProtocol {
- _ = skip.Set(otherKey, struct{}{})
- final = append(final, []*Template{other})
- continue
+ if template.RequestsDNS[0].IsClusterable() {
+ hash := template.RequestsDNS[0].TmplClusterKey()
+ if dns[hash] == nil {
+ dns[hash] = []*Template{}
+ }
+ dns[hash] = append(dns[hash], template)
+ } else {
+ final = append(final, []*Template{template})
}
- switch templateType {
- case types.DNSProtocol:
- if len(other.RequestsDNS) != 1 {
- continue
- } else if template.RequestsDNS[0].CanCluster(other.RequestsDNS[0]) {
- _ = skip.Set(otherKey, struct{}{})
- cluster = append(cluster, other)
- }
- case types.HTTPProtocol:
- if len(other.RequestsHTTP) != 1 {
- continue
- } else if template.RequestsHTTP[0].CanCluster(other.RequestsHTTP[0]) {
- _ = skip.Set(otherKey, struct{}{})
- cluster = append(cluster, other)
+ case len(template.RequestsHTTP) == 1:
+ if template.RequestsHTTP[0].IsClusterable() {
+ hash := template.RequestsHTTP[0].TmplClusterKey()
+ if http[hash] == nil {
+ http[hash] = []*Template{}
}
- case types.SSLProtocol:
- if len(other.RequestsSSL) != 1 {
- continue
- } else if template.RequestsSSL[0].CanCluster(other.RequestsSSL[0]) {
- _ = skip.Set(otherKey, struct{}{})
- cluster = append(cluster, other)
+ http[hash] = append(http[hash], template)
+ } else {
+ final = append(final, []*Template{template})
+ }
+ case len(template.RequestsSSL) == 1:
+ if template.RequestsSSL[0].IsClusterable() {
+ hash := template.RequestsSSL[0].TmplClusterKey()
+ if ssl[hash] == nil {
+ ssl[hash] = []*Template{}
}
+ ssl[hash] = append(ssl[hash], template)
+ } else {
+ final = append(final, []*Template{template})
}
- }
- if len(cluster) > 0 {
- cluster = append(cluster, template)
- final = append(final, cluster)
- } else {
+ default:
final = append(final, []*Template{template})
}
}
+
+ // add all clusterd templates
+ for _, templates := range http {
+ final = append(final, templates)
+ }
+ for _, templates := range dns {
+ final = append(final, templates)
+ }
+ for _, templates := range ssl {
+ final = append(final, templates)
+ }
+
return final
}
diff --git a/pkg/utils/utils.go b/pkg/utils/utils.go
index 0d95b13f46..dfe37ae99e 100644
--- a/pkg/utils/utils.go
+++ b/pkg/utils/utils.go
@@ -2,12 +2,16 @@ package utils
import (
"errors"
+ "fmt"
"io"
"net/url"
"strings"
+ "github.com/cespare/xxhash"
"github.com/projectdiscovery/nuclei/v3/pkg/catalog"
"github.com/projectdiscovery/retryablehttp-go"
+ mapsutil "github.com/projectdiscovery/utils/maps"
+ "golang.org/x/exp/constraints"
)
func IsBlank(value string) bool {
@@ -57,3 +61,13 @@ func StringSliceContains(slice []string, item string) bool {
}
return false
}
+
+// MapHash generates a hash for any give map
+func MapHash[K constraints.Ordered, V any](m map[K]V) uint64 {
+ keys := mapsutil.GetSortedKeys(m)
+ var sb strings.Builder
+ for _, k := range keys {
+ sb.WriteString(fmt.Sprintf("%v:%v\n", k, m[k]))
+ }
+ return xxhash.Sum64([]byte(sb.String()))
+}
From 2b89a2e1a03e835c00ecd7d43a4bfafdd9ab0b9c Mon Sep 17 00:00:00 2001
From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com>
Date: Mon, 1 Jul 2024 05:56:31 +0000
Subject: [PATCH 074/283] chore(deps): bump
github.com/projectdiscovery/retryablehttp-go
Bumps [github.com/projectdiscovery/retryablehttp-go](https://github.com/projectdiscovery/retryablehttp-go) from 1.0.66 to 1.0.67.
- [Release notes](https://github.com/projectdiscovery/retryablehttp-go/releases)
- [Commits](https://github.com/projectdiscovery/retryablehttp-go/compare/v1.0.66...v1.0.67)
---
updated-dependencies:
- dependency-name: github.com/projectdiscovery/retryablehttp-go
dependency-type: direct:production
update-type: version-update:semver-patch
...
Signed-off-by: dependabot[bot]
---
go.mod | 8 ++++----
go.sum | 13 ++++++-------
2 files changed, 10 insertions(+), 11 deletions(-)
diff --git a/go.mod b/go.mod
index be2c0ca475..8d9324053a 100644
--- a/go.mod
+++ b/go.mod
@@ -21,11 +21,11 @@ require (
github.com/pkg/errors v0.9.1
github.com/projectdiscovery/clistats v0.0.20
github.com/projectdiscovery/fastdialer v0.1.5
- github.com/projectdiscovery/hmap v0.0.47
+ github.com/projectdiscovery/hmap v0.0.48
github.com/projectdiscovery/interactsh v1.2.0
github.com/projectdiscovery/rawhttp v0.1.51
github.com/projectdiscovery/retryabledns v1.0.65
- github.com/projectdiscovery/retryablehttp-go v1.0.66
+ github.com/projectdiscovery/retryablehttp-go v1.0.67
github.com/projectdiscovery/yamldoc-go v1.0.4
github.com/remeh/sizedwaitgroup v1.0.0
github.com/rs/xid v1.5.0
@@ -58,6 +58,7 @@ require (
github.com/aws/aws-sdk-go-v2/credentials v1.13.27
github.com/aws/aws-sdk-go-v2/feature/s3/manager v1.11.72
github.com/aws/aws-sdk-go-v2/service/s3 v1.37.0
+ github.com/cespare/xxhash v1.1.0
github.com/charmbracelet/glamour v0.6.0
github.com/clbanning/mxj/v2 v2.7.0
github.com/denisenkom/go-mssqldb v0.12.3
@@ -94,7 +95,7 @@ require (
github.com/projectdiscovery/tlsx v1.1.6
github.com/projectdiscovery/uncover v1.0.8
github.com/projectdiscovery/useragent v0.0.56
- github.com/projectdiscovery/utils v0.1.4
+ github.com/projectdiscovery/utils v0.1.5
github.com/projectdiscovery/wappalyzergo v0.1.6
github.com/redis/go-redis/v9 v9.1.0
github.com/seh-msft/burpxml v1.0.1
@@ -129,7 +130,6 @@ require (
github.com/buger/jsonparser v1.1.1 // indirect
github.com/bytedance/sonic v1.9.1 // indirect
github.com/cenkalti/backoff/v4 v4.2.1 // indirect
- github.com/cespare/xxhash v1.1.0 // indirect
github.com/cespare/xxhash/v2 v2.2.0 // indirect
github.com/cheggaaa/pb/v3 v3.1.4 // indirect
github.com/chenzhuoyu/base64x v0.0.0-20221115062448-fe3a3abad311 // indirect
diff --git a/go.sum b/go.sum
index 5cda424843..6c48dd914f 100644
--- a/go.sum
+++ b/go.sum
@@ -351,7 +351,6 @@ github.com/go-git/gcfg v1.5.1-0.20230307220236-3a3c6141e376/go.mod h1:an3vInlBmS
github.com/go-git/go-billy/v5 v5.5.0 h1:yEY4yhzCDuMGSv83oGxiBotRzhwhNr8VZyphhiu+mTU=
github.com/go-git/go-billy/v5 v5.5.0/go.mod h1:hmexnoNsr2SJU1Ju67OaNz5ASJY3+sHgFRpCtpDCKow=
github.com/go-git/go-git-fixtures/v4 v4.3.2-0.20231010084843-55a94097c399 h1:eMje31YglSBqCdIqdhKBW8lokaMrL3uTkpGYlE2OOT4=
-github.com/go-git/go-git-fixtures/v4 v4.3.2-0.20231010084843-55a94097c399/go.mod h1:1OCfN199q1Jm3HZlxleg+Dw/mwps2Wbk9frAWm+4FII=
github.com/go-git/go-git/v5 v5.11.0 h1:XIZc1p+8YzypNr34itUfSvYJcv+eYdTnTvOZ2vD3cA4=
github.com/go-git/go-git/v5 v5.11.0/go.mod h1:6GFcX2P3NM7FPBfpePbpLd21XxsgdAt+lKqXmCUiUCY=
github.com/go-gl/glfw v0.0.0-20190409004039-e6da0acd62b1/go.mod h1:vR7hzQXu2zJy9AVAgeJqvqgH9Q5CA+iKCZ2gyEVpxRU=
@@ -852,8 +851,8 @@ github.com/projectdiscovery/gostruct v0.0.2 h1:s8gP8ApugGM4go1pA+sVlPDXaWqNP5BBD
github.com/projectdiscovery/gostruct v0.0.2/go.mod h1:H86peL4HKwMXcQQtEa6lmC8FuD9XFt6gkNR0B/Mu5PE=
github.com/projectdiscovery/gozero v0.0.2 h1:8fJeaCjxL9tpm33uG/RsCQs6HGM/NE6eA3cjkilRQ+E=
github.com/projectdiscovery/gozero v0.0.2/go.mod h1:d8bZvDWW07LWNYWrwjZ4OO1I0cpkfqaysyDfSs9ibK8=
-github.com/projectdiscovery/hmap v0.0.47 h1:NdakfvlFruevnOvehdllofg/hc0CQn2WQaLHaRcsGAk=
-github.com/projectdiscovery/hmap v0.0.47/go.mod h1:/9V6EnyTY33hAy71Quox6Ggu9ZkKp36tgepBh/NzNSg=
+github.com/projectdiscovery/hmap v0.0.48 h1:pbxoDcwZWSdaqZ3QgaG5Aw1G1qEFN2zeQco6eWeROAo=
+github.com/projectdiscovery/hmap v0.0.48/go.mod h1:KMSbPJoIgI+9PkG6alZ0INEMRiIGnND128RUjxiY+8I=
github.com/projectdiscovery/httpx v1.6.3 h1:TSu3zEKOHGFfMsGeB0lZEFRl6Tq4nPuIgKDXi9NnYpg=
github.com/projectdiscovery/httpx v1.6.3/go.mod h1:9FmsmkT71aTa4WnOyBIPzB8I9c0QAJ9PSVUqs2NkHRQ=
github.com/projectdiscovery/interactsh v1.2.0 h1:Al6jHiR+Usl9egYJDLJaWNHOcH8Rugk8gWMasc8Cmw8=
@@ -876,8 +875,8 @@ github.com/projectdiscovery/rdap v0.9.1-0.20221108103045-9865884d1917 h1:m03X4gB
github.com/projectdiscovery/rdap v0.9.1-0.20221108103045-9865884d1917/go.mod h1:JxXtZC9e195awe7EynrcnBJmFoad/BNDzW9mzFkK8Sg=
github.com/projectdiscovery/retryabledns v1.0.65 h1:wOtMuVkrcrGkzaT8Jixb3BtUcJODXLRz+z9EjftmFsQ=
github.com/projectdiscovery/retryabledns v1.0.65/go.mod h1:lf3xRdFhGh55fWVlzK1oAaDPrsKliqH3gGvsFNAOkCk=
-github.com/projectdiscovery/retryablehttp-go v1.0.66 h1:dY+sGVf/O8CvKcSdTx0rxJoKG+HyOScz5T89TijZohU=
-github.com/projectdiscovery/retryablehttp-go v1.0.66/go.mod h1:3QJt06j4hEZVpJOr31u49I3mDnqbAzhcjJX8Yvn/xnY=
+github.com/projectdiscovery/retryablehttp-go v1.0.67 h1:3jCiREsQEWJN2fG3D+i46zJNkq5+GtrLGf+Q77jGgrE=
+github.com/projectdiscovery/retryablehttp-go v1.0.67/go.mod h1:LXTRQdTOOmMLJDbw5R9fyYgQKLjwS/l5tVt2wP09FBI=
github.com/projectdiscovery/sarif v0.0.1 h1:C2Tyj0SGOKbCLgHrx83vaE6YkzXEVrMXYRGLkKCr/us=
github.com/projectdiscovery/sarif v0.0.1/go.mod h1:cEYlDu8amcPf6b9dSakcz2nNnJsoz4aR6peERwV+wuQ=
github.com/projectdiscovery/stringsutil v0.0.2 h1:uzmw3IVLJSMW1kEg8eCStG/cGbYYZAja8BH3LqqJXMA=
@@ -888,8 +887,8 @@ github.com/projectdiscovery/uncover v1.0.8 h1:nE8bYJuwhqk0BEMRQRhNbjPGMy40A9gkiM
github.com/projectdiscovery/uncover v1.0.8/go.mod h1:1SwsNqjwMkJEzJQ7lQr5AHIdYd+BQlyqhO5IwIYmIAM=
github.com/projectdiscovery/useragent v0.0.56 h1:KpUtOgq1vUerPrgRFZAIr1hF0lDxA2RF6YkT2PLwSoQ=
github.com/projectdiscovery/useragent v0.0.56/go.mod h1:LAh9vudxXPA5JfNN3+1g/Unqvulq0BDkjxK7IrE+e/s=
-github.com/projectdiscovery/utils v0.1.4 h1:VSULN/RBEXbkgWuhrDey84ZDnSXtYMSev9p+QHXXDeI=
-github.com/projectdiscovery/utils v0.1.4/go.mod h1:mXs6OOeG9l/dVchjB2PGvQO3+wuMiE14Y/kmHeKogoM=
+github.com/projectdiscovery/utils v0.1.5 h1:vjD2ZVtVEEiWNIKIFWkWSxaHnw8wGln8Lyfru6E+BW0=
+github.com/projectdiscovery/utils v0.1.5/go.mod h1:NhjK2eVeoXLIZIYBJ2Z7yl6V4sB6Xr5Bzs1GdAqvJZ4=
github.com/projectdiscovery/wappalyzergo v0.1.6 h1:4iZDEnytsnAqjQgD5K6lmAzVx++KSQeMCDfomDtQM1M=
github.com/projectdiscovery/wappalyzergo v0.1.6/go.mod h1:wBYGKmA5BQp/NWsAy1q/jSH8N1LHWQ/LV26DuR+KzPM=
github.com/projectdiscovery/yamldoc-go v1.0.4 h1:eZoESapnMw6WAHiVgRwNqvbJEfNHEH148uthhFbG5jE=
From 68cac06818ee0fc200a5c6031c46eb17cce184eb Mon Sep 17 00:00:00 2001
From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com>
Date: Mon, 1 Jul 2024 06:25:36 +0000
Subject: [PATCH 075/283] chore(deps): bump
github.com/projectdiscovery/useragent
Bumps [github.com/projectdiscovery/useragent](https://github.com/projectdiscovery/useragent) from 0.0.56 to 0.0.57.
- [Release notes](https://github.com/projectdiscovery/useragent/releases)
- [Commits](https://github.com/projectdiscovery/useragent/compare/v0.0.56...v0.0.57)
---
updated-dependencies:
- dependency-name: github.com/projectdiscovery/useragent
dependency-type: direct:production
update-type: version-update:semver-patch
...
Signed-off-by: dependabot[bot]
---
go.mod | 2 +-
go.sum | 5 +++--
2 files changed, 4 insertions(+), 3 deletions(-)
diff --git a/go.mod b/go.mod
index 8d9324053a..4dba575ed8 100644
--- a/go.mod
+++ b/go.mod
@@ -94,7 +94,7 @@ require (
github.com/projectdiscovery/sarif v0.0.1
github.com/projectdiscovery/tlsx v1.1.6
github.com/projectdiscovery/uncover v1.0.8
- github.com/projectdiscovery/useragent v0.0.56
+ github.com/projectdiscovery/useragent v0.0.57
github.com/projectdiscovery/utils v0.1.5
github.com/projectdiscovery/wappalyzergo v0.1.6
github.com/redis/go-redis/v9 v9.1.0
diff --git a/go.sum b/go.sum
index 6c48dd914f..8a5ed5c0de 100644
--- a/go.sum
+++ b/go.sum
@@ -351,6 +351,7 @@ github.com/go-git/gcfg v1.5.1-0.20230307220236-3a3c6141e376/go.mod h1:an3vInlBmS
github.com/go-git/go-billy/v5 v5.5.0 h1:yEY4yhzCDuMGSv83oGxiBotRzhwhNr8VZyphhiu+mTU=
github.com/go-git/go-billy/v5 v5.5.0/go.mod h1:hmexnoNsr2SJU1Ju67OaNz5ASJY3+sHgFRpCtpDCKow=
github.com/go-git/go-git-fixtures/v4 v4.3.2-0.20231010084843-55a94097c399 h1:eMje31YglSBqCdIqdhKBW8lokaMrL3uTkpGYlE2OOT4=
+github.com/go-git/go-git-fixtures/v4 v4.3.2-0.20231010084843-55a94097c399/go.mod h1:1OCfN199q1Jm3HZlxleg+Dw/mwps2Wbk9frAWm+4FII=
github.com/go-git/go-git/v5 v5.11.0 h1:XIZc1p+8YzypNr34itUfSvYJcv+eYdTnTvOZ2vD3cA4=
github.com/go-git/go-git/v5 v5.11.0/go.mod h1:6GFcX2P3NM7FPBfpePbpLd21XxsgdAt+lKqXmCUiUCY=
github.com/go-gl/glfw v0.0.0-20190409004039-e6da0acd62b1/go.mod h1:vR7hzQXu2zJy9AVAgeJqvqgH9Q5CA+iKCZ2gyEVpxRU=
@@ -885,8 +886,8 @@ github.com/projectdiscovery/tlsx v1.1.6 h1:iw2zwKbd2+kRQ8J1G4dLmS0CLyemd/tKz1Uzc
github.com/projectdiscovery/tlsx v1.1.6/go.mod h1:s7SRRFdrwIZBK/RXXZi4CR/CubqFSvp8h5Bk1srEZIo=
github.com/projectdiscovery/uncover v1.0.8 h1:nE8bYJuwhqk0BEMRQRhNbjPGMy40A9gkiMk5xq6U4u0=
github.com/projectdiscovery/uncover v1.0.8/go.mod h1:1SwsNqjwMkJEzJQ7lQr5AHIdYd+BQlyqhO5IwIYmIAM=
-github.com/projectdiscovery/useragent v0.0.56 h1:KpUtOgq1vUerPrgRFZAIr1hF0lDxA2RF6YkT2PLwSoQ=
-github.com/projectdiscovery/useragent v0.0.56/go.mod h1:LAh9vudxXPA5JfNN3+1g/Unqvulq0BDkjxK7IrE+e/s=
+github.com/projectdiscovery/useragent v0.0.57 h1:y6ML3GVJ6Sq+nhqk2gY9aD0JNaKzfpunJPjPN40eVN4=
+github.com/projectdiscovery/useragent v0.0.57/go.mod h1:iChqL4rGApUsdXhAzItirEpMvYYOlruieZE+nqoKpCc=
github.com/projectdiscovery/utils v0.1.5 h1:vjD2ZVtVEEiWNIKIFWkWSxaHnw8wGln8Lyfru6E+BW0=
github.com/projectdiscovery/utils v0.1.5/go.mod h1:NhjK2eVeoXLIZIYBJ2Z7yl6V4sB6Xr5Bzs1GdAqvJZ4=
github.com/projectdiscovery/wappalyzergo v0.1.6 h1:4iZDEnytsnAqjQgD5K6lmAzVx++KSQeMCDfomDtQM1M=
From 1e3cfd75ea852be2b925223eb051a917b50d7938 Mon Sep 17 00:00:00 2001
From: Tarun Koyalwar
Date: Wed, 3 Jul 2024 18:26:59 +0530
Subject: [PATCH 076/283] fix issue with ldap search func
---
pkg/js/libs/ldap/ldap.go | 11 +++++++++--
1 file changed, 9 insertions(+), 2 deletions(-)
diff --git a/pkg/js/libs/ldap/ldap.go b/pkg/js/libs/ldap/ldap.go
index 5f03611cb2..31afa232d0 100644
--- a/pkg/js/libs/ldap/ldap.go
+++ b/pkg/js/libs/ldap/ldap.go
@@ -205,11 +205,18 @@ func (c *Client) AuthenticateWithNTLMHash(username, hash string) {
// ```
func (c *Client) Search(filter string, attributes ...string) []map[string][]string {
c.nj.Require(c.conn != nil, "no existing connection")
+ c.nj.Require(c.BaseDN != "", "base dn cannot be empty")
+ c.nj.Require(len(attributes) > 0, "attributes cannot be empty")
res, err := c.conn.Search(
ldap.NewSearchRequest(
- c.BaseDN, ldap.ScopeWholeSubtree, ldap.NeverDerefAliases,
- 0, 0, false, filter, attributes, nil,
+ "",
+ ldap.ScopeBaseObject,
+ ldap.NeverDerefAliases,
+ 0, 0, false,
+ filter,
+ attributes,
+ nil,
),
)
c.nj.HandleError(err, "ldap search request failed")
From 3ff085c5edb5948be6ca98749a5d7086fb26c5d1 Mon Sep 17 00:00:00 2001
From: Tarun Koyalwar
Date: Wed, 3 Jul 2024 18:30:32 +0530
Subject: [PATCH 077/283] bump version to dev
---
pkg/catalog/config/constants.go | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/pkg/catalog/config/constants.go b/pkg/catalog/config/constants.go
index 0ee9f437bb..113cb11509 100644
--- a/pkg/catalog/config/constants.go
+++ b/pkg/catalog/config/constants.go
@@ -31,7 +31,7 @@ const (
CLIConfigFileName = "config.yaml"
ReportingConfigFilename = "reporting-config.yaml"
// Version is the current version of nuclei
- Version = `v3.2.9`
+ Version = `v3.3.0-dev`
// Directory Names of custom templates
CustomS3TemplatesDirName = "s3"
CustomGitHubTemplatesDirName = "github"
From ec89a23bc5fb9372dccbab5c307f3be489017e5f Mon Sep 17 00:00:00 2001
From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com>
Date: Mon, 8 Jul 2024 05:57:41 +0000
Subject: [PATCH 078/283] chore(deps): bump github.com/projectdiscovery/asnmap
from 1.1.0 to 1.1.1
Bumps [github.com/projectdiscovery/asnmap](https://github.com/projectdiscovery/asnmap) from 1.1.0 to 1.1.1.
- [Release notes](https://github.com/projectdiscovery/asnmap/releases)
- [Changelog](https://github.com/projectdiscovery/asnmap/blob/main/.goreleaser.yml)
- [Commits](https://github.com/projectdiscovery/asnmap/compare/v1.1.0...v1.1.1)
---
updated-dependencies:
- dependency-name: github.com/projectdiscovery/asnmap
dependency-type: indirect
update-type: version-update:semver-patch
...
Signed-off-by: dependabot[bot]
---
go.mod | 2 +-
go.sum | 4 ++--
2 files changed, 3 insertions(+), 3 deletions(-)
diff --git a/go.mod b/go.mod
index 4dba575ed8..b0089222c9 100644
--- a/go.mod
+++ b/go.mod
@@ -200,7 +200,7 @@ require (
github.com/pierrec/lz4/v4 v4.1.21 // indirect
github.com/pjbgf/sha1cd v0.3.0 // indirect
github.com/pkg/browser v0.0.0-20240102092130-5ac0b6a4141c // indirect
- github.com/projectdiscovery/asnmap v1.1.0 // indirect
+ github.com/projectdiscovery/asnmap v1.1.1 // indirect
github.com/projectdiscovery/cdncheck v1.0.9 // indirect
github.com/projectdiscovery/freeport v0.0.5 // indirect
github.com/projectdiscovery/ldapserver v1.0.2-0.20240219154113-dcc758ebc0cb // indirect
diff --git a/go.sum b/go.sum
index 8a5ed5c0de..901dabeb2b 100644
--- a/go.sum
+++ b/go.sum
@@ -826,8 +826,8 @@ github.com/power-devops/perfstat v0.0.0-20210106213030-5aafc221ea8c h1:ncq/mPwQF
github.com/power-devops/perfstat v0.0.0-20210106213030-5aafc221ea8c/go.mod h1:OmDBASR4679mdNQnz2pUhc2G8CO2JrUAVFDRBDP/hJE=
github.com/praetorian-inc/fingerprintx v1.1.9 h1:zWbG/Fdan0s/dvXkeaHb/CdFTz/yEEzrAF4iCzok3r8=
github.com/praetorian-inc/fingerprintx v1.1.9/go.mod h1:k6EJIHe/Da4DH5e4JuoZHe+qSGq/KPUmXGaK+xW74OI=
-github.com/projectdiscovery/asnmap v1.1.0 h1:ynvbLB5cNpyQ2+k9IP0Rpla+0JmCJpd3mw6KLAW13m0=
-github.com/projectdiscovery/asnmap v1.1.0/go.mod h1:QNjBnGLxUBEZAgaYk/Av5cjKKWFY3i/FOfoIWCUApoY=
+github.com/projectdiscovery/asnmap v1.1.1 h1:ImJiKIaACOT7HPx4Pabb5dksolzaFYsD1kID2iwsDqI=
+github.com/projectdiscovery/asnmap v1.1.1/go.mod h1:QT7jt9nQanj+Ucjr9BqGr1Q2veCCKSAVyUzLXfEcQ60=
github.com/projectdiscovery/blackrock v0.0.1 h1:lHQqhaaEFjgf5WkuItbpeCZv2DUIE45k0VbGJyft6LQ=
github.com/projectdiscovery/blackrock v0.0.1/go.mod h1:ANUtjDfaVrqB453bzToU+YB4cUbvBRpLvEwoWIwlTss=
github.com/projectdiscovery/cdncheck v1.0.9 h1:BS15gzj9gb5AVSKqTDzPamfSgStu7nJQOocUvrssFlg=
From 5cb32a431060edc0100df5d2816ce9a1fe3fbd92 Mon Sep 17 00:00:00 2001
From: Ramana Reddy <90540245+RamanaReddy0M@users.noreply.github.com>
Date: Mon, 8 Jul 2024 16:21:11 +0530
Subject: [PATCH 079/283] Fix target loading with input-mode (#5369)
---
pkg/input/provider/http/multiformat.go | 1 +
1 file changed, 1 insertion(+)
diff --git a/pkg/input/provider/http/multiformat.go b/pkg/input/provider/http/multiformat.go
index ee95079ef4..651c3656f6 100644
--- a/pkg/input/provider/http/multiformat.go
+++ b/pkg/input/provider/http/multiformat.go
@@ -68,6 +68,7 @@ func (i *HttpInputProvider) Iterate(callback func(value *contextargs.MetaInput)
err := i.format.Parse(i.inputFile, func(request *types.RequestResponse) bool {
return callback(&contextargs.MetaInput{
ReqResp: request,
+ Input: request.URL.String(),
})
})
if err != nil {
From 1c24ceda4548676f2c116f031d51f483bd3cae69 Mon Sep 17 00:00:00 2001
From: Mzack9999
Date: Mon, 8 Jul 2024 13:38:29 +0200
Subject: [PATCH 080/283] adding automatic service check on query (#5291)
* adding automatic service check on query
* automatic service check
---
pkg/js/libs/mysql/mysql.go | 30 ++++++++++++++++++++++++++++++
pkg/js/libs/postgres/postgres.go | 32 ++++++++++++++++++++++++++++----
2 files changed, 58 insertions(+), 4 deletions(-)
diff --git a/pkg/js/libs/mysql/mysql.go b/pkg/js/libs/mysql/mysql.go
index 0fd2f24200..4566059260 100644
--- a/pkg/js/libs/mysql/mysql.go
+++ b/pkg/js/libs/mysql/mysql.go
@@ -36,6 +36,7 @@ type (
// const isMySQL = mysql.IsMySQL('acme.com', 3306);
// ```
func (c *MySQLClient) IsMySQL(host string, port int) (bool, error) {
+ // todo: why this is exposed? Service fingerprint should be automatic
return memoizedisMySQL(host, port)
}
@@ -77,6 +78,16 @@ func (c *MySQLClient) Connect(host string, port int, username, password string)
// host is not valid according to network policy
return false, protocolstate.ErrHostDenied.Msgf(host)
}
+
+ // executing queries implies the remote mysql service
+ ok, err := c.IsMySQL(host, port)
+ if err != nil {
+ return false, err
+ }
+ if !ok {
+ return false, fmt.Errorf("not a mysql service")
+ }
+
dsn, err := BuildDSN(MySQLOptions{
Host: host,
Port: port,
@@ -182,6 +193,16 @@ func (c *MySQLClient) ExecuteQueryWithOpts(opts MySQLOptions, query string) (*ut
// host is not valid according to network policy
return nil, protocolstate.ErrHostDenied.Msgf(opts.Host)
}
+
+ // executing queries implies the remote mysql service
+ ok, err := c.IsMySQL(opts.Host, opts.Port)
+ if err != nil {
+ return nil, err
+ }
+ if !ok {
+ return nil, fmt.Errorf("not a mysql service")
+ }
+
dsn, err := BuildDSN(opts)
if err != nil {
return nil, err
@@ -220,6 +241,15 @@ func (c *MySQLClient) ExecuteQueryWithOpts(opts MySQLOptions, query string) (*ut
// log(to_json(result));
// ```
func (c *MySQLClient) ExecuteQuery(host string, port int, username, password, query string) (*utils.SQLResult, error) {
+ // executing queries implies the remote mysql service
+ ok, err := c.IsMySQL(host, port)
+ if err != nil {
+ return nil, err
+ }
+ if !ok {
+ return nil, fmt.Errorf("not a mysql service")
+ }
+
return c.ExecuteQueryWithOpts(MySQLOptions{
Host: host,
Port: port,
diff --git a/pkg/js/libs/postgres/postgres.go b/pkg/js/libs/postgres/postgres.go
index f20ea85a73..e3093c49d3 100644
--- a/pkg/js/libs/postgres/postgres.go
+++ b/pkg/js/libs/postgres/postgres.go
@@ -37,6 +37,7 @@ type (
// const isPostgres = postgres.IsPostgres('acme.com', 5432);
// ```
func (c *PGClient) IsPostgres(host string, port int) (bool, error) {
+ // todo: why this is exposed? Service fingerprint should be automatic
return memoizedisPostgres(host, port)
}
@@ -74,6 +75,13 @@ func isPostgres(host string, port int) (bool, error) {
// const connected = client.Connect('acme.com', 5432, 'username', 'password');
// ```
func (c *PGClient) Connect(host string, port int, username, password string) (bool, error) {
+ ok, err := c.IsPostgres(host, port)
+ if err != nil {
+ return false, err
+ }
+ if !ok {
+ return false, fmt.Errorf("not a postgres service")
+ }
return memoizedconnect(host, port, username, password, "postgres")
}
@@ -88,6 +96,14 @@ func (c *PGClient) Connect(host string, port int, username, password string) (bo
// log(to_json(result));
// ```
func (c *PGClient) ExecuteQuery(host string, port int, username, password, dbName, query string) (*utils.SQLResult, error) {
+ ok, err := c.IsPostgres(host, port)
+ if err != nil {
+ return nil, err
+ }
+ if !ok {
+ return nil, fmt.Errorf("not a postgres service")
+ }
+
return memoizedexecuteQuery(host, port, username, password, dbName, query)
}
@@ -129,6 +145,14 @@ func executeQuery(host string, port int, username string, password string, dbNam
// const connected = client.ConnectWithDB('acme.com', 5432, 'username', 'password', 'dbname');
// ```
func (c *PGClient) ConnectWithDB(host string, port int, username, password, dbName string) (bool, error) {
+ ok, err := c.IsPostgres(host, port)
+ if err != nil {
+ return false, err
+ }
+ if !ok {
+ return false, fmt.Errorf("not a postgres service")
+ }
+
return memoizedconnect(host, port, username, password, dbName)
}
@@ -149,10 +173,10 @@ func connect(host string, port int, username string, password string, dbName str
defer cancel()
db := pg.Connect(&pg.Options{
- Addr: target,
- User: username,
- Password: password,
- Database: dbName,
+ Addr: target,
+ User: username,
+ Password: password,
+ Database: dbName,
Dialer: func(network, addr string) (net.Conn, error) {
return protocolstate.Dialer.Dial(context.Background(), network, addr)
},
From c9a9bd3bfc47f56505bc8312c72589157de517d6 Mon Sep 17 00:00:00 2001
From: Tarun Koyalwar <45962551+tarunKoyalwar@users.noreply.github.com>
Date: Wed, 10 Jul 2024 20:43:22 +0530
Subject: [PATCH 081/283] include cname in http output if available (#5389)
---
pkg/protocols/http/request.go | 21 +++++++++++++++++++++
1 file changed, 21 insertions(+)
diff --git a/pkg/protocols/http/request.go b/pkg/protocols/http/request.go
index 7e5e069348..c84db691f7 100644
--- a/pkg/protocols/http/request.go
+++ b/pkg/protocols/http/request.go
@@ -844,6 +844,8 @@ func (request *Request) executeRequest(input *contextargs.Context, generatedRequ
outputEvent["ip"] = input.MetaInput.CustomIP
} else {
outputEvent["ip"] = protocolstate.Dialer.GetDialedIP(hostname)
+ // try getting cname
+ request.addCNameIfAvailable(hostname, outputEvent)
}
if len(generatedRequest.interactshURLs) > 0 {
@@ -940,6 +942,8 @@ func (request *Request) executeRequest(input *contextargs.Context, generatedRequ
outputEvent["ip"] = input.MetaInput.CustomIP
} else {
outputEvent["ip"] = protocolstate.Dialer.GetDialedIP(hostname)
+ // try getting cname
+ request.addCNameIfAvailable(hostname, outputEvent)
}
if request.options.Interactsh != nil {
request.options.Interactsh.MakePlaceholders(generatedRequest.interactshURLs, outputEvent)
@@ -1031,6 +1035,23 @@ func (request *Request) validateNFixEvent(input *contextargs.Context, gr *genera
}
}
+// addCNameIfAvailable adds the cname to the event if available
+func (request *Request) addCNameIfAvailable(hostname string, outputEvent map[string]interface{}) {
+ data, err := protocolstate.Dialer.GetDNSData(hostname)
+ if err == nil {
+ switch len(data.CNAME) {
+ case 0:
+ return
+ case 1:
+ outputEvent["cname"] = data.CNAME[0]
+ default:
+ // add 1st and put others in cname_all
+ outputEvent["cname"] = data.CNAME[0]
+ outputEvent["cname_all"] = data.CNAME
+ }
+ }
+}
+
// handleSignature of the http request
func (request *Request) handleSignature(generatedRequest *generatedRequest) error {
switch request.Signature.Value {
From 3c2af0e47c0fea68cd5d2100b9ebb88aa3090ff4 Mon Sep 17 00:00:00 2001
From: mzack9999
Date: Fri, 12 Jul 2024 17:23:44 +0200
Subject: [PATCH 082/283] fixing many data races
---
.../protocols/code/pre-condition.yaml | 2 +-
.../protocols/code/py-env-var.yaml | 2 +-
integration_tests/protocols/code/py-file.yaml | 2 +-
.../protocols/code/py-interactsh.yaml | 2 +-
.../protocols/code/py-snippet.yaml | 2 +-
.../workflow/code-template-1.yaml | 2 +-
.../workflow/code-template-2.yaml | 2 +-
pkg/input/provider/http/multiformat.go | 8 ++--
pkg/input/provider/list/hmap.go | 40 +++++++++++++------
pkg/input/provider/list/hmap_test.go | 6 +--
pkg/input/provider/simple.go | 12 ++++--
.../common/contextargs/contextargs.go | 4 +-
pkg/protocols/common/contextargs/metainput.go | 17 ++++++--
pkg/scan/scan_context.go | 3 ++
14 files changed, 69 insertions(+), 35 deletions(-)
diff --git a/integration_tests/protocols/code/pre-condition.yaml b/integration_tests/protocols/code/pre-condition.yaml
index 1c44e9579a..2190ffecbe 100644
--- a/integration_tests/protocols/code/pre-condition.yaml
+++ b/integration_tests/protocols/code/pre-condition.yaml
@@ -23,4 +23,4 @@ code:
- type: dsl
dsl:
- true
-# digest: 490a004630440220192fb8f704b078c2885047b85ac1a0491be86485c033a976d201599683a35aab0220604b1c3781e9d97079d0e5c23c18e6a2d87493c8e2b930536e692ee7d06e9247:4a3eb6b4988d95847d4203be25ed1d46
\ No newline at end of file
+# digest: 4a0a0047304502200307590191cb7c766b6c21e5777d345bdddf7adf9d6da8f7d336d585d9ac4a8b022100fd30fb0c7722778eb3d861d60e721d805925b8d8df2b979ef2104c35ec57d5cb:4a3eb6b4988d95847d4203be25ed1d46
\ No newline at end of file
diff --git a/integration_tests/protocols/code/py-env-var.yaml b/integration_tests/protocols/code/py-env-var.yaml
index 9ff947c208..067c183bb7 100644
--- a/integration_tests/protocols/code/py-env-var.yaml
+++ b/integration_tests/protocols/code/py-env-var.yaml
@@ -20,4 +20,4 @@ code:
- type: word
words:
- "hello from input baz"
-# digest: 4a0a00473045022033f72f1b9d5143f58a2dc79c2597000f34080251ac3702c36c3fad00917dfeeb0221009ba05c715c9e2e36dba471be6c0106a09ae3822d8a3e9e4bcf377e9f4a395a01:4a3eb6b4988d95847d4203be25ed1d46
\ No newline at end of file
+# digest: 4a0a0047304502203fe1d7d52bc2a41886d576a90c82c3be42078baaa4b46e1f3d8519665d6f88b202210081feb82c41150c5b218e226fc4f299ded19f42ba01ef34ba60b0634b4ea6ee12:4a3eb6b4988d95847d4203be25ed1d46
\ No newline at end of file
diff --git a/integration_tests/protocols/code/py-file.yaml b/integration_tests/protocols/code/py-file.yaml
index ad69371d05..3521ec902d 100644
--- a/integration_tests/protocols/code/py-file.yaml
+++ b/integration_tests/protocols/code/py-file.yaml
@@ -18,4 +18,4 @@ code:
- type: word
words:
- "hello from input"
-# digest: 4a0a004730450220377128cb11d9f6f0fee1f4dbd841e46783de26e90a216fa55a7609ee2bc823c60221009166ee0f85e3a1811588ab19e73ea96ab3d582dc8180dbcbbad0ea9ab7e9025d:4a3eb6b4988d95847d4203be25ed1d46
\ No newline at end of file
+# digest: 4b0a00483046022100afb5ebff14a40e7f9b679ffc4d93ce7849e33eb398ebb47f2e757cd24831f9dd02210089ffa21b2763e99ebce95dfc5b91e1e62da4ccdc9d2ad5c48584fa350ba335af:4a3eb6b4988d95847d4203be25ed1d46
\ No newline at end of file
diff --git a/integration_tests/protocols/code/py-interactsh.yaml b/integration_tests/protocols/code/py-interactsh.yaml
index 76d14efb2e..0ccab7a7cd 100644
--- a/integration_tests/protocols/code/py-interactsh.yaml
+++ b/integration_tests/protocols/code/py-interactsh.yaml
@@ -26,4 +26,4 @@ code:
part: interactsh_protocol
words:
- "http"
-# digest: 4b0a00483046022100d472d50bd83117d334f5217c7a40dcdf34138e90029eaace51697d902296bf37022100a393b49420a96f60d6d89b79b5135ee2233b2468d374851890eea114b08195d1:4a3eb6b4988d95847d4203be25ed1d46
\ No newline at end of file
+# digest: 4b0a00483046022100939f83e74d43932a5bd792b1fd2c100eec2df60f2b2a8dd56b5c8ef5faa92b17022100f93031b0de373af7d78e623968ea5a2d67c4561ef70e3e6da15aef7e5c853115:4a3eb6b4988d95847d4203be25ed1d46
\ No newline at end of file
diff --git a/integration_tests/protocols/code/py-snippet.yaml b/integration_tests/protocols/code/py-snippet.yaml
index 4837fa7e0e..5e36c1a673 100644
--- a/integration_tests/protocols/code/py-snippet.yaml
+++ b/integration_tests/protocols/code/py-snippet.yaml
@@ -21,4 +21,4 @@ code:
- type: word
words:
- "hello from input"
-# digest: 4b0a004830460221008886054bb5dd6345e434e30f31c8fddce3c484a4f33aa6321b5185675866029d022100d188a83d0fde029f8b586061c65ab72b43755c3fb10fdd59501bb9bbadbb1ff7:4a3eb6b4988d95847d4203be25ed1d46
\ No newline at end of file
+# digest: 4a0a00473045022100b8e676ce0c57b60c233a0203539dec20457bbb5f1790d351a5d45405b6668b2602204b1f2fa18e7db099f05329009597ceb2d9b7337562c1a676e8d50ea2f1c6fcbe:4a3eb6b4988d95847d4203be25ed1d46
\ No newline at end of file
diff --git a/integration_tests/workflow/code-template-1.yaml b/integration_tests/workflow/code-template-1.yaml
index d41a1a6957..81a9773ae0 100644
--- a/integration_tests/workflow/code-template-1.yaml
+++ b/integration_tests/workflow/code-template-1.yaml
@@ -19,4 +19,4 @@ code:
regex:
- 'hello from (.*)'
group: 1
-# digest: 490a0046304402202c63d47bb0acdd40b3b852d95490d492ff5741b84071b2a8a40371be7797c13602202b6b977e157edf2ef70a402a2e57d4eb5a67c5ca91f0a2f9a10a966e8485ebaf:4a3eb6b4988d95847d4203be25ed1d46
\ No newline at end of file
+# digest: 490a00463044022050da011362cf08c2cb81e812c7f86d7282afe0562d4bf00d390f1300d19bc910022029e9d305da69e941ac18797645aecb217abde6557f891e141301b48e89a3c0cd:4a3eb6b4988d95847d4203be25ed1d46
\ No newline at end of file
diff --git a/integration_tests/workflow/code-template-2.yaml b/integration_tests/workflow/code-template-2.yaml
index ddd7d36a4e..1fb9805501 100644
--- a/integration_tests/workflow/code-template-2.yaml
+++ b/integration_tests/workflow/code-template-2.yaml
@@ -18,4 +18,4 @@ code:
- type: word
words:
- "hello from first"
-# digest: 490a00463044022025661eab353b7f359c0d428a86b6287545d7f759375e8025cc8c9c77b616ca6502200bc2c019059622df3c88e7caa6dd7d1fb9b956010aa0de2ee2b9f7dd0a3c4954:4a3eb6b4988d95847d4203be25ed1d46
\ No newline at end of file
+# digest: 4b0a00483046022100b3b8759c0df028455eb59b1433ac240e5d4604b011bb0c63680bd3cc159ac6f0022100f44aa11b640d11ad0e2902897f4eb51666ab3cd83c31dfd2590f6e43391e39b0:4a3eb6b4988d95847d4203be25ed1d46
\ No newline at end of file
diff --git a/pkg/input/provider/http/multiformat.go b/pkg/input/provider/http/multiformat.go
index 651c3656f6..d58970fec5 100644
--- a/pkg/input/provider/http/multiformat.go
+++ b/pkg/input/provider/http/multiformat.go
@@ -66,10 +66,10 @@ func (i *HttpInputProvider) Count() int64 {
// Iterate over all inputs in order
func (i *HttpInputProvider) Iterate(callback func(value *contextargs.MetaInput) bool) {
err := i.format.Parse(i.inputFile, func(request *types.RequestResponse) bool {
- return callback(&contextargs.MetaInput{
- ReqResp: request,
- Input: request.URL.String(),
- })
+ metaInput := contextargs.NewMetaInput()
+ metaInput.ReqResp = request
+ metaInput.Input = request.URL.String()
+ return callback(metaInput)
})
if err != nil {
gologger.Warning().Msgf("Could not parse input file while iterating: %s\n", err)
diff --git a/pkg/input/provider/list/hmap.go b/pkg/input/provider/list/hmap.go
index f7e3bd14c6..edf3729193 100644
--- a/pkg/input/provider/list/hmap.go
+++ b/pkg/input/provider/list/hmap.go
@@ -122,7 +122,7 @@ func (i *ListInputProvider) Iterate(callback func(value *contextargs.MetaInput)
})
}
callbackFunc := func(k, _ []byte) error {
- metaInput := &contextargs.MetaInput{}
+ metaInput := contextargs.NewMetaInput()
if err := metaInput.Unmarshal(string(k)); err != nil {
return err
}
@@ -153,14 +153,16 @@ func (i *ListInputProvider) Set(value string) {
}
return fmt.Sprintf("got empty hostname for %v skipping ip selection", URL)
})
- metaInput := &contextargs.MetaInput{Input: URL}
+ metaInput := contextargs.NewMetaInput()
+ metaInput.Input = URL
i.setItem(metaInput)
return
}
// Check if input is ip or hostname
if iputil.IsIP(urlx.Hostname()) {
- metaInput := &contextargs.MetaInput{Input: URL}
+ metaInput := contextargs.NewMetaInput()
+ metaInput.Input = URL
i.setItem(metaInput)
return
}
@@ -181,7 +183,9 @@ func (i *ListInputProvider) Set(value string) {
if ip == "" {
continue
}
- metaInput := &contextargs.MetaInput{Input: value, CustomIP: ip}
+ metaInput := contextargs.NewMetaInput()
+ metaInput.Input = value
+ metaInput.CustomIP = ip
i.setItem(metaInput)
}
return
@@ -211,11 +215,13 @@ func (i *ListInputProvider) Set(value string) {
}
for _, ip := range ips {
+ metaInput := contextargs.NewMetaInput()
if ip != "" {
- metaInput := &contextargs.MetaInput{Input: URL, CustomIP: ip}
+ metaInput.Input = URL
+ metaInput.CustomIP = ip
i.setItem(metaInput)
} else {
- metaInput := &contextargs.MetaInput{Input: URL}
+ metaInput.Input = URL
i.setItem(metaInput)
}
}
@@ -353,7 +359,8 @@ func (i *ListInputProvider) scanInputFromReader(reader io.Reader) {
// isExcluded checks if a URL is in the exclusion list
func (i *ListInputProvider) isExcluded(URL string) bool {
- metaInput := &contextargs.MetaInput{Input: URL}
+ metaInput := contextargs.NewMetaInput()
+ metaInput.Input = URL
key, err := metaInput.MarshalString()
if err != nil {
gologger.Warning().Msgf("%s\n", err)
@@ -378,14 +385,16 @@ func (i *ListInputProvider) Del(value string) {
}
return fmt.Sprintf("got empty hostname for %v skipping ip selection", URL)
})
- metaInput := &contextargs.MetaInput{Input: URL}
+ metaInput := contextargs.NewMetaInput()
+ metaInput.Input = URL
i.delItem(metaInput)
return
}
// Check if input is ip or hostname
if iputil.IsIP(urlx.Hostname()) {
- metaInput := &contextargs.MetaInput{Input: URL}
+ metaInput := contextargs.NewMetaInput()
+ metaInput.Input = URL
i.delItem(metaInput)
return
}
@@ -406,7 +415,9 @@ func (i *ListInputProvider) Del(value string) {
if ip == "" {
continue
}
- metaInput := &contextargs.MetaInput{Input: value, CustomIP: ip}
+ metaInput := contextargs.NewMetaInput()
+ metaInput.Input = value
+ metaInput.CustomIP = ip
i.delItem(metaInput)
}
return
@@ -436,11 +447,13 @@ func (i *ListInputProvider) Del(value string) {
}
for _, ip := range ips {
+ metaInput := contextargs.NewMetaInput()
if ip != "" {
- metaInput := &contextargs.MetaInput{Input: URL, CustomIP: ip}
+ metaInput.Input = URL
+ metaInput.CustomIP = ip
i.delItem(metaInput)
} else {
- metaInput := &contextargs.MetaInput{Input: URL}
+ metaInput.Input = URL
i.delItem(metaInput)
}
}
@@ -514,7 +527,8 @@ func (i *ListInputProvider) addTargets(targets []string) {
func (i *ListInputProvider) removeTargets(targets []string) {
for _, target := range targets {
- metaInput := &contextargs.MetaInput{Input: target}
+ metaInput := contextargs.NewMetaInput()
+ metaInput.Input = target
i.delItem(metaInput)
}
}
diff --git a/pkg/input/provider/list/hmap_test.go b/pkg/input/provider/list/hmap_test.go
index e8a7e8eea8..cd28b247a1 100644
--- a/pkg/input/provider/list/hmap_test.go
+++ b/pkg/input/provider/list/hmap_test.go
@@ -40,7 +40,7 @@ func Test_expandCIDR(t *testing.T) {
// scan
got := []string{}
input.hostMap.Scan(func(k, _ []byte) error {
- var metainput contextargs.MetaInput
+ metainput := contextargs.NewMetaInput()
if err := metainput.Unmarshal(string(k)); err != nil {
return err
}
@@ -141,7 +141,7 @@ func Test_scanallips_normalizeStoreInputValue(t *testing.T) {
// scan
got := []string{}
input.hostMap.Scan(func(k, v []byte) error {
- var metainput contextargs.MetaInput
+ metainput := contextargs.NewMetaInput()
if err := metainput.Unmarshal(string(k)); err != nil {
return err
}
@@ -184,7 +184,7 @@ func Test_expandASNInputValue(t *testing.T) {
// scan the hmap
got := []string{}
input.hostMap.Scan(func(k, v []byte) error {
- var metainput contextargs.MetaInput
+ metainput := contextargs.NewMetaInput()
if err := metainput.Unmarshal(string(k)); err != nil {
return err
}
diff --git a/pkg/input/provider/simple.go b/pkg/input/provider/simple.go
index 71b280c2f5..c85f7871b9 100644
--- a/pkg/input/provider/simple.go
+++ b/pkg/input/provider/simple.go
@@ -43,7 +43,9 @@ func (s *SimpleInputProvider) Iterate(callback func(value *contextargs.MetaInput
// Set adds an item to the input provider
func (s *SimpleInputProvider) Set(value string) {
- s.Inputs = append(s.Inputs, &contextargs.MetaInput{Input: value})
+ metaInput := contextargs.NewMetaInput()
+ metaInput.Input = value
+ s.Inputs = append(s.Inputs, metaInput)
}
// SetWithProbe adds an item to the input provider with HTTP probing
@@ -52,13 +54,17 @@ func (s *SimpleInputProvider) SetWithProbe(value string, probe types.InputLivene
if err != nil {
return err
}
- s.Inputs = append(s.Inputs, &contextargs.MetaInput{Input: probedValue})
+ metaInput := contextargs.NewMetaInput()
+ metaInput.Input = probedValue
+ s.Inputs = append(s.Inputs, metaInput)
return nil
}
// SetWithExclusions adds an item to the input provider if it doesn't match any of the exclusions
func (s *SimpleInputProvider) SetWithExclusions(value string) error {
- s.Inputs = append(s.Inputs, &contextargs.MetaInput{Input: value})
+ metaInput := contextargs.NewMetaInput()
+ metaInput.Input = value
+ s.Inputs = append(s.Inputs, metaInput)
return nil
}
diff --git a/pkg/protocols/common/contextargs/contextargs.go b/pkg/protocols/common/contextargs/contextargs.go
index ed004ae370..af55595498 100644
--- a/pkg/protocols/common/contextargs/contextargs.go
+++ b/pkg/protocols/common/contextargs/contextargs.go
@@ -50,9 +50,11 @@ func NewWithInput(ctx context.Context, input string) *Context {
if err != nil {
gologger.Error().Msgf("contextargs: could not create cookie jar: %s\n", err)
}
+ metaInput := NewMetaInput()
+ metaInput.Input = input
return &Context{
ctx: ctx,
- MetaInput: &MetaInput{Input: input},
+ MetaInput: metaInput,
CookieJar: jar,
args: &mapsutil.SyncLockMap[string, interface{}]{
Map: make(map[string]interface{}),
diff --git a/pkg/protocols/common/contextargs/metainput.go b/pkg/protocols/common/contextargs/metainput.go
index 72eac6247d..b3c0dabe7a 100644
--- a/pkg/protocols/common/contextargs/metainput.go
+++ b/pkg/protocols/common/contextargs/metainput.go
@@ -6,6 +6,7 @@ import (
"fmt"
"net"
"strings"
+ "sync"
jsoniter "github.com/json-iterator/go"
"github.com/projectdiscovery/nuclei/v3/pkg/input/types"
@@ -24,6 +25,12 @@ type MetaInput struct {
// ReqResp is the raw request for the input
ReqResp *types.RequestResponse `json:"raw-request,omitempty"`
+
+ mu *sync.Mutex
+}
+
+func NewMetaInput() *MetaInput {
+ return &MetaInput{mu: &sync.Mutex{}}
}
func (metaInput *MetaInput) marshalToBuffer() (bytes.Buffer, error) {
@@ -135,10 +142,9 @@ func (metaInput *MetaInput) Unmarshal(data string) error {
}
func (metaInput *MetaInput) Clone() *MetaInput {
- input := &MetaInput{
- Input: metaInput.Input,
- CustomIP: metaInput.CustomIP,
- }
+ input := NewMetaInput()
+ input.Input = metaInput.Input
+ input.CustomIP = metaInput.CustomIP
if metaInput.ReqResp != nil {
input.ReqResp = metaInput.ReqResp.Clone()
}
@@ -160,6 +166,9 @@ func (metaInput *MetaInput) GetScanHash(templateId string) string {
// there may be some cases where metainput is changed ex: while executing self-contained template etc
// but that totally changes the scanID/hash so to avoid that we compute hash only once
// and reuse it for all subsequent calls
+ metaInput.mu.Lock()
+ defer metaInput.mu.Unlock()
+
if metaInput.hash == "" {
var rawRequest string
if metaInput.ReqResp != nil {
diff --git a/pkg/scan/scan_context.go b/pkg/scan/scan_context.go
index 69fbfde4c0..b8f59ac7d4 100644
--- a/pkg/scan/scan_context.go
+++ b/pkg/scan/scan_context.go
@@ -77,6 +77,9 @@ func (s *ScanContext) LogEvent(e *output.InternalWrappedEvent) {
s.events = append(s.events, e)
}
+ e.RLock()
+ defer e.RUnlock()
+
s.results = append(s.results, e.Results...)
}
From 2bd68993e7923bae684d16b5fda0b1711ff466e3 Mon Sep 17 00:00:00 2001
From: mzack9999
Date: Fri, 12 Jul 2024 19:45:49 +0200
Subject: [PATCH 083/283] fixing loader issue
---
cmd/integration-test/loader.go | 18 ++++++++++++++++--
pkg/testutils/integration.go | 10 +++++++---
2 files changed, 23 insertions(+), 5 deletions(-)
diff --git a/cmd/integration-test/loader.go b/cmd/integration-test/loader.go
index 2c2beb4cd9..8e5ea40ff8 100644
--- a/cmd/integration-test/loader.go
+++ b/cmd/integration-test/loader.go
@@ -172,7 +172,14 @@ func (h *nonExistentTemplateList) Execute(nonExistingTemplateList string) error
ts := httptest.NewServer(router)
defer ts.Close()
- _, err := testutils.RunNucleiBareArgsAndGetResults(debug, nil, "-target", ts.URL, "-template-url", ts.URL+"/404")
+ configFileData := `remote-template-domain: [ "` + ts.Listener.Addr().String() + `" ]`
+ err := os.WriteFile("test-config.yaml", []byte(configFileData), permissionutil.ConfigFilePermission)
+ if err != nil {
+ return err
+ }
+ defer os.Remove("test-config.yaml")
+
+ _, err = testutils.RunNucleiBareArgsAndGetResults(debug, nil, "-target", ts.URL, "-template-url", ts.URL+"/404", "-config", "test-config.yaml")
if err == nil {
return fmt.Errorf("expected error for nonexisting workflow url")
}
@@ -188,7 +195,14 @@ func (h *nonExistentWorkflowList) Execute(nonExistingWorkflowList string) error
ts := httptest.NewServer(router)
defer ts.Close()
- _, err := testutils.RunNucleiBareArgsAndGetResults(debug, nil, "-target", ts.URL, "-workflow-url", ts.URL+"/404")
+ configFileData := `remote-template-domain: [ "` + ts.Listener.Addr().String() + `" ]`
+ err := os.WriteFile("test-config.yaml", []byte(configFileData), permissionutil.ConfigFilePermission)
+ if err != nil {
+ return err
+ }
+ defer os.Remove("test-config.yaml")
+
+ _, err = testutils.RunNucleiBareArgsAndGetResults(debug, nil, "-target", ts.URL, "-workflow-url", ts.URL+"/404", "-config", "test-config.yaml")
if err == nil {
return fmt.Errorf("expected error for nonexisting workflow url")
}
diff --git a/pkg/testutils/integration.go b/pkg/testutils/integration.go
index 8b5a71b1ab..8bd78a53b6 100644
--- a/pkg/testutils/integration.go
+++ b/pkg/testutils/integration.go
@@ -14,6 +14,7 @@ import (
"github.com/gobwas/ws"
"github.com/julienschmidt/httprouter"
+ "github.com/projectdiscovery/utils/conversion"
)
// ExtraArgs
@@ -74,9 +75,7 @@ func RunNucleiBareArgsAndGetResults(debug bool, env []string, extra ...string) (
if debug {
fmt.Println(string(data))
}
- if len(data) < 1 && err != nil {
- return nil, fmt.Errorf("%v: %v", err.Error(), string(data))
- }
+ dataStr := conversion.String(data)
var parts []string
items := strings.Split(string(data), "\n")
for _, i := range items {
@@ -84,6 +83,11 @@ func RunNucleiBareArgsAndGetResults(debug bool, env []string, extra ...string) (
parts = append(parts, i)
}
}
+
+ if (dataStr == "" || len(parts) == 0) && err != nil {
+ return nil, fmt.Errorf("%v: %v", err.Error(), dataStr)
+ }
+
return parts, nil
}
From 860507bbe9ecef90f3e78c04bf3fe9f6eb5a9d36 Mon Sep 17 00:00:00 2001
From: mzack9999
Date: Sun, 14 Jul 2024 23:19:42 +0200
Subject: [PATCH 084/283] fixing tests
---
pkg/testutils/integration.go | 46 +++++++++++++++++++++++-------------
1 file changed, 29 insertions(+), 17 deletions(-)
diff --git a/pkg/testutils/integration.go b/pkg/testutils/integration.go
index 8bd78a53b6..83a6105cc2 100644
--- a/pkg/testutils/integration.go
+++ b/pkg/testutils/integration.go
@@ -71,11 +71,14 @@ func RunNucleiBareArgsAndGetResults(debug bool, env []string, extra ...string) (
} else {
cmd.Args = append(cmd.Args, "-silent")
}
- data, err := cmd.Output()
+ output, err := cmd.Output()
+ var data string
+ if len(output) > 0 {
+ data = conversion.String(output)
+ }
if debug {
- fmt.Println(string(data))
+ fmt.Println(data)
}
- dataStr := conversion.String(data)
var parts []string
items := strings.Split(string(data), "\n")
for _, i := range items {
@@ -84,8 +87,8 @@ func RunNucleiBareArgsAndGetResults(debug bool, env []string, extra ...string) (
}
}
- if (dataStr == "" || len(parts) == 0) && err != nil {
- return nil, fmt.Errorf("%v: %v", err.Error(), dataStr)
+ if (data == "" || len(parts) == 0) && err != nil {
+ return nil, fmt.Errorf("%v: %v", err.Error(), data)
}
return parts, nil
@@ -102,20 +105,25 @@ func RunNucleiWithArgsAndGetResults(debug bool, args ...string) ([]string, error
} else {
cmd.Args = append(cmd.Args, "-silent")
}
- data, err := cmd.Output()
- if debug {
- fmt.Println(string(data))
+ output, err := cmd.Output()
+ var data string
+ if len(output) > 0 {
+ data = conversion.String(output)
}
- if len(data) < 1 && err != nil {
- return nil, fmt.Errorf("%v: %v", err.Error(), string(data))
+ if debug {
+ fmt.Println(data)
}
var parts []string
- items := strings.Split(string(data), "\n")
+ items := strings.Split(data, "\n")
for _, i := range items {
if i != "" {
parts = append(parts, i)
}
}
+
+ if (data == "" || len(parts) == 0) && err != nil {
+ return nil, fmt.Errorf("%v: %v", err.Error(), data)
+ }
return parts, nil
}
@@ -168,20 +176,24 @@ func RunNucleiArgsWithEnvAndGetResults(debug bool, env []string, extra ...string
} else {
cmd.Args = append(cmd.Args, "-silent")
}
- data, err := cmd.Output()
- if debug {
- fmt.Println(string(data))
+ dataOutput, err := cmd.Output()
+ var data string
+ if len(dataOutput) > 0 {
+ data = conversion.String(dataOutput)
}
- if len(data) < 1 && err != nil {
- return nil, fmt.Errorf("%v: %v", err.Error(), string(data))
+ if debug {
+ fmt.Println(data)
}
var parts []string
- items := strings.Split(string(data), "\n")
+ items := strings.Split(data, "\n")
for _, i := range items {
if i != "" {
parts = append(parts, i)
}
}
+ if (data == "" || len(parts) == 0) && err != nil {
+ return nil, fmt.Errorf("%v: %v", err.Error(), data)
+ }
return parts, nil
}
From cdb39d851bee9bbac7773815708d0d0d05dcd0ad Mon Sep 17 00:00:00 2001
From: mzack9999
Date: Sun, 14 Jul 2024 23:45:28 +0200
Subject: [PATCH 085/283] trim space
---
pkg/testutils/integration.go | 16 ++++++++++------
1 file changed, 10 insertions(+), 6 deletions(-)
diff --git a/pkg/testutils/integration.go b/pkg/testutils/integration.go
index 83a6105cc2..7d6e7cebfa 100644
--- a/pkg/testutils/integration.go
+++ b/pkg/testutils/integration.go
@@ -74,7 +74,7 @@ func RunNucleiBareArgsAndGetResults(debug bool, env []string, extra ...string) (
output, err := cmd.Output()
var data string
if len(output) > 0 {
- data = conversion.String(output)
+ data = strings.TrimSpace(conversion.String(output))
}
if debug {
fmt.Println(data)
@@ -108,7 +108,7 @@ func RunNucleiWithArgsAndGetResults(debug bool, args ...string) ([]string, error
output, err := cmd.Output()
var data string
if len(output) > 0 {
- data = conversion.String(output)
+ data = strings.TrimSpace(conversion.String(output))
}
if debug {
fmt.Println(data)
@@ -139,12 +139,16 @@ func RunNucleiArgsAndGetErrors(debug bool, env []string, extra ...string) ([]str
cmd.Args = append(cmd.Args, "-allow-local-file-access")
cmd.Args = append(cmd.Args, "-nc") // disable color
cmd.Env = append(cmd.Env, ExtraEnvVars...)
- data, err := cmd.CombinedOutput()
+ dataOutput, err := cmd.CombinedOutput()
if debug {
- fmt.Println(string(data))
+ fmt.Println(string(dataOutput))
+ }
+ var data string
+ if len(dataOutput) > 0 {
+ data = strings.TrimSpace(conversion.String(dataOutput))
}
results := []string{}
- for _, v := range strings.Split(string(data), "\n") {
+ for _, v := range strings.Split(data, "\n") {
line := strings.TrimSpace(v)
switch {
case strings.HasPrefix(line, "[ERR]"):
@@ -179,7 +183,7 @@ func RunNucleiArgsWithEnvAndGetResults(debug bool, env []string, extra ...string
dataOutput, err := cmd.Output()
var data string
if len(dataOutput) > 0 {
- data = conversion.String(dataOutput)
+ data = strings.TrimSpace(conversion.String(dataOutput))
}
if debug {
fmt.Println(data)
From d7997e47ecbbb72ea0757143438f7e157e8770d8 Mon Sep 17 00:00:00 2001
From: mzack9999
Date: Sun, 14 Jul 2024 23:47:05 +0200
Subject: [PATCH 086/283] trim space
---
pkg/testutils/integration.go | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/pkg/testutils/integration.go b/pkg/testutils/integration.go
index 7d6e7cebfa..c8a67e41cb 100644
--- a/pkg/testutils/integration.go
+++ b/pkg/testutils/integration.go
@@ -80,7 +80,7 @@ func RunNucleiBareArgsAndGetResults(debug bool, env []string, extra ...string) (
fmt.Println(data)
}
var parts []string
- items := strings.Split(string(data), "\n")
+ items := strings.Split(data, "\n")
for _, i := range items {
if i != "" {
parts = append(parts, i)
From d4e81fd9e6a9f3c6de2b925fec029637a2aec702 Mon Sep 17 00:00:00 2001
From: Ramana Reddy <90540245+RamanaReddy0M@users.noreply.github.com>
Date: Mon, 15 Jul 2024 00:28:30 -0700
Subject: [PATCH 087/283] register goja func to check udp port (#5397)
* register goja func to check port with network param
* register goja func to check udp port
---
pkg/js/devtools/scrapefuncs/README.md | 6 +++++-
pkg/js/global/scripts.go | 23 ++++++++++++++++++++++-
2 files changed, 27 insertions(+), 2 deletions(-)
diff --git a/pkg/js/devtools/scrapefuncs/README.md b/pkg/js/devtools/scrapefuncs/README.md
index de7c29ce2c..ecde2d5ab4 100644
--- a/pkg/js/devtools/scrapefuncs/README.md
+++ b/pkg/js/devtools/scrapefuncs/README.md
@@ -42,7 +42,11 @@ Description: getNetworkPort registers defaultPort and returns defaultPort if it
Name: isPortOpen
Signatures: "isPortOpen(host string, port string, [timeout int]) bool"
-Description: isPortOpen checks if given port is open on host. timeout is optional and defaults to 5 seconds
+Description: isPortOpen checks if given TCP port is open on host. timeout is optional and defaults to 5 seconds
+
+Name: isUDPPortOpen
+Signatures: "isUDPPortOpen(host string, port string, [timeout int]) bool"
+Description: isUDPPortOpen checks if the given UDP port is open on the host. Timeout is optional and defaults to 5 seconds.
Name: ToBytes
Signatures: "ToBytes(...interface{}) []byte"
diff --git a/pkg/js/global/scripts.go b/pkg/js/global/scripts.go
index 64d6b25c17..c6771fadf3 100644
--- a/pkg/js/global/scripts.go
+++ b/pkg/js/global/scripts.go
@@ -109,7 +109,7 @@ func initBuiltInFunc(runtime *goja.Runtime) {
Signatures: []string{
"isPortOpen(host string, port string, [timeout int]) bool",
},
- Description: "isPortOpen checks if given port is open on host. timeout is optional and defaults to 5 seconds",
+ Description: "isPortOpen checks if given TCP port is open on host. timeout is optional and defaults to 5 seconds",
FuncDecl: func(host string, port string, timeout ...int) (bool, error) {
timeoutInSec := 5
if len(timeout) > 0 {
@@ -124,6 +124,27 @@ func initBuiltInFunc(runtime *goja.Runtime) {
},
})
+ _ = gojs.RegisterFuncWithSignature(runtime, gojs.FuncOpts{
+ Name: "isUDPPortOpen",
+ Signatures: []string{
+ "isUDPPortOpen(host string, port string, [timeout int]) bool",
+ },
+ Description: "isUDPPortOpen checks if the given UDP port is open on the host. Timeout is optional and defaults to 5 seconds.",
+ FuncDecl: func(host string, port string, timeout ...int) (bool, error) {
+ timeoutInSec := 5
+ if len(timeout) > 0 {
+ timeoutInSec = timeout[0]
+ }
+ conn, err := net.DialTimeout("udp", net.JoinHostPort(host, port), time.Duration(timeoutInSec)*time.Second)
+ if err != nil {
+ return false, err
+ }
+ _ = conn.Close()
+
+ return true, nil
+ },
+ })
+
_ = gojs.RegisterFuncWithSignature(runtime, gojs.FuncOpts{
Name: "ToBytes",
Signatures: []string{
From f080d614c3c7dfc0e9656fb8913620de5fb0d690 Mon Sep 17 00:00:00 2001
From: Dogan Can Bakir <65292895+dogancanbakir@users.noreply.github.com>
Date: Mon, 15 Jul 2024 13:27:15 +0300
Subject: [PATCH 088/283] introduce timeouts config in types.Options (#5228)
* introduce timeout variants
* update instances and add codeexectimeout
* fix test
* default to 10s
* minor
* make timeouts pluggable and rename
* remove residual code
---------
Co-authored-by: Tarun Koyalwar
---
cmd/nuclei/main.go | 2 -
pkg/js/compiler/compiler.go | 24 ++----
pkg/js/compiler/compiler_test.go | 19 +++--
pkg/js/compiler/init.go | 11 +--
pkg/protocols/code/code.go | 18 ++---
pkg/protocols/common/protocolstate/state.go | 4 +-
pkg/protocols/http/http.go | 2 +-
.../http/httpclientpool/clientpool.go | 23 ++----
pkg/protocols/http/request.go | 2 +-
pkg/protocols/http/request_annotations.go | 6 +-
pkg/protocols/javascript/js.go | 20 +++--
pkg/protocols/network/request.go | 4 +-
pkg/templates/compile_test.go | 18 ++---
pkg/testutils/testutils.go | 23 +++---
pkg/tmplexec/flow/flow_executor_test.go | 18 ++---
pkg/types/types.go | 78 +++++++++++++++++--
16 files changed, 151 insertions(+), 121 deletions(-)
diff --git a/cmd/nuclei/main.go b/cmd/nuclei/main.go
index 5aea16337c..2d2be0aa1a 100644
--- a/cmd/nuclei/main.go
+++ b/cmd/nuclei/main.go
@@ -297,7 +297,6 @@ on extensive configurability, massive extensibility and ease of use.`)
flagSet.BoolVarP(&options.ShowMatchLine, "show-match-line", "sml", false, "show match lines for file templates, works with extractors only"),
flagSet.BoolVar(&options.ZTLS, "ztls", false, "use ztls library with autofallback to standard one for tls13 [Deprecated] autofallback to ztls is enabled by default"), //nolint:all
flagSet.StringVar(&options.SNI, "sni", "", "tls sni hostname to use (default: input domain name)"),
- flagSet.DurationVarP(&options.DialerTimeout, "dialer-timeout", "dt", 0, "timeout for network requests."),
flagSet.DurationVarP(&options.DialerKeepAlive, "dialer-keep-alive", "dka", 0, "keep-alive duration for network requests."),
flagSet.BoolVarP(&options.AllowLocalFileAccess, "allow-local-file-access", "lfa", false, "allows file (payload) access anywhere on the system"),
flagSet.BoolVarP(&options.RestrictLocalNetworkAccess, "restrict-local-network-access", "lna", false, "blocks connections to the local / private network"),
@@ -306,7 +305,6 @@ on extensive configurability, massive extensibility and ease of use.`)
flagSet.StringVarP(&options.SourceIP, "source-ip", "sip", "", "source ip address to use for network scan"),
flagSet.IntVarP(&options.ResponseReadSize, "response-size-read", "rsr", 0, "max response size to read in bytes"),
flagSet.IntVarP(&options.ResponseSaveSize, "response-size-save", "rss", unitutils.Mega, "max response size to read in bytes"),
- flagSet.DurationVarP(&options.ResponseReadTimeout, "response-read-timeout", "rrt", time.Duration(5*time.Second), "response read timeout in seconds"),
flagSet.CallbackVar(resetCallback, "reset", "reset removes all nuclei configuration and data files (including nuclei-templates)"),
flagSet.BoolVarP(&options.TlsImpersonate, "tls-impersonate", "tlsi", false, "enable experimental client hello (ja3) tls randomization"),
flagSet.StringVarP(&options.HttpApiEndpoint, "http-api-endpoint", "hae", "", "experimental http api endpoint"),
diff --git a/pkg/js/compiler/compiler.go b/pkg/js/compiler/compiler.go
index 367bd013b2..f50e44ff2a 100644
--- a/pkg/js/compiler/compiler.go
+++ b/pkg/js/compiler/compiler.go
@@ -4,11 +4,11 @@ package compiler
import (
"context"
"fmt"
- "time"
"github.com/dop251/goja"
"github.com/projectdiscovery/nuclei/v3/pkg/protocols/common/generators"
+ "github.com/projectdiscovery/nuclei/v3/pkg/types"
contextutil "github.com/projectdiscovery/utils/context"
"github.com/projectdiscovery/utils/errkit"
stringsutil "github.com/projectdiscovery/utils/strings"
@@ -38,13 +38,13 @@ type ExecuteOptions struct {
// Cleanup is extra cleanup function to be called after execution
Cleanup func(runtime *goja.Runtime)
- /// Timeout for this script execution
- Timeout int
// Source is original source of the script
Source *string
Context context.Context
+ TimeoutVariants *types.Timeouts
+
// Manually exported objects
exports map[string]interface{}
}
@@ -79,15 +79,6 @@ func (e ExecuteResult) GetSuccess() bool {
return val
}
-// Execute executes a script with the default options.
-func (c *Compiler) Execute(code string, args *ExecuteArgs) (ExecuteResult, error) {
- p, err := WrapScriptNCompile(code, false)
- if err != nil {
- return nil, err
- }
- return c.ExecuteWithOptions(p, args, &ExecuteOptions{Context: context.Background()})
-}
-
// ExecuteWithOptions executes a script with the provided options.
func (c *Compiler) ExecuteWithOptions(program *goja.Program, args *ExecuteArgs, opts *ExecuteOptions) (ExecuteResult, error) {
if opts == nil {
@@ -106,14 +97,9 @@ func (c *Compiler) ExecuteWithOptions(program *goja.Program, args *ExecuteArgs,
// merge all args into templatectx
args.TemplateCtx = generators.MergeMaps(args.TemplateCtx, args.Args)
- if opts.Timeout <= 0 || opts.Timeout > 180 {
- // some js scripts can take longer time so allow configuring timeout
- // from template but keep it within sane limits (180s)
- opts.Timeout = JsProtocolTimeout
- }
-
// execute with context and timeout
- ctx, cancel := context.WithTimeoutCause(opts.Context, time.Duration(opts.Timeout)*time.Second, ErrJSExecDeadline)
+
+ ctx, cancel := context.WithTimeoutCause(opts.Context, opts.TimeoutVariants.JsCompilerExecutionTimeout, ErrJSExecDeadline)
defer cancel()
// execute the script
results, err := contextutil.ExecFuncWithTwoReturns(ctx, func() (val goja.Value, err error) {
diff --git a/pkg/js/compiler/compiler_test.go b/pkg/js/compiler/compiler_test.go
index ca09e7782e..3a43a4016e 100644
--- a/pkg/js/compiler/compiler_test.go
+++ b/pkg/js/compiler/compiler_test.go
@@ -1,11 +1,14 @@
package compiler
import (
+ "context"
"strings"
"testing"
+ "time"
"github.com/projectdiscovery/gologger"
"github.com/projectdiscovery/gologger/levels"
+ "github.com/projectdiscovery/nuclei/v3/pkg/types"
)
func TestNewCompilerConsoleDebug(t *testing.T) {
@@ -18,23 +21,19 @@ func TestNewCompilerConsoleDebug(t *testing.T) {
})
compiler := New()
- _, err := compiler.Execute("console.log('hello world');", NewExecuteArgs())
+ p, err := WrapScriptNCompile("console.log('hello world');", false)
if err != nil {
t.Fatal(err)
}
- if !strings.HasSuffix(gotString, "hello world") {
- t.Fatalf("console.log not working, got=%v", gotString)
- }
-}
-func TestExecuteResultGetSuccess(t *testing.T) {
- compiler := New()
- result, err := compiler.Execute("1+1 == 2", NewExecuteArgs())
+ _, err = compiler.ExecuteWithOptions(p, NewExecuteArgs(), &ExecuteOptions{Context: context.Background(),
+ TimeoutVariants: &types.Timeouts{JsCompilerExecutionTimeout: time.Duration(20) * time.Second}},
+ )
if err != nil {
t.Fatal(err)
}
- if result.GetSuccess() != true {
- t.Fatalf("expected true, got=%v", result.GetSuccess())
+ if !strings.HasSuffix(gotString, "hello world") {
+ t.Fatalf("console.log not working, got=%v", gotString)
}
}
diff --git a/pkg/js/compiler/init.go b/pkg/js/compiler/init.go
index 8a171dc3da..92301df5e4 100644
--- a/pkg/js/compiler/init.go
+++ b/pkg/js/compiler/init.go
@@ -7,26 +7,17 @@ import (
// jsprotocolInit
var (
- // Per Execution Javascript timeout in seconds
- JsProtocolTimeout = 10
PoolingJsVmConcurrency = 100
NonPoolingVMConcurrency = 20
- JsTimeoutMultiplier = 1.5
)
// Init initializes the javascript protocol
func Init(opts *types.Options) error {
- if opts.Timeout < 10 {
- // keep existing 10s timeout
- return nil
- }
+
if opts.JsConcurrency < 100 {
// 100 is reasonable default
opts.JsConcurrency = 100
}
- // we have dialer timeout set to 10s so js needs to be at least
- // 15s to return the actual error if not it will be a dialer timeout
- JsProtocolTimeout = int(float64(opts.Timeout) * JsTimeoutMultiplier)
PoolingJsVmConcurrency = opts.JsConcurrency
PoolingJsVmConcurrency -= NonPoolingVMConcurrency
return nil
diff --git a/pkg/protocols/code/code.go b/pkg/protocols/code/code.go
index f103f50d0c..7736113dca 100644
--- a/pkg/protocols/code/code.go
+++ b/pkg/protocols/code/code.go
@@ -37,8 +37,7 @@ import (
)
const (
- pythonEnvRegex = `os\.getenv\(['"]([^'"]+)['"]\)`
- TimeoutMultiplier = 6 // timeout multiplier for code protocol
+ pythonEnvRegex = `os\.getenv\(['"]([^'"]+)['"]\)`
)
var (
@@ -179,9 +178,6 @@ func (request *Request) ExecuteWithResults(input *contextargs.Context, dynamicVa
metaSrc.AddVariable(gozerotypes.Variable{Name: name, Value: v})
}
- // set timeout using multiplier
- timeout := TimeoutMultiplier * request.options.Options.Timeout
-
if request.PreCondition != "" {
if request.options.Options.Debug || request.options.Options.DebugRequests {
gologger.Debug().Msgf("[%s] Executing Precondition for Code request\n", request.TemplateID)
@@ -199,11 +195,11 @@ func (request *Request) ExecuteWithResults(input *contextargs.Context, dynamicVa
result, err := request.options.JsCompiler.ExecuteWithOptions(request.preConditionCompiled, args,
&compiler.ExecuteOptions{
- Timeout: timeout,
- Source: &request.PreCondition,
- Callback: registerPreConditionFunctions,
- Cleanup: cleanUpPreConditionFunctions,
- Context: input.Context(),
+ TimeoutVariants: request.options.Options.GetTimeouts(),
+ Source: &request.PreCondition,
+ Callback: registerPreConditionFunctions,
+ Cleanup: cleanUpPreConditionFunctions,
+ Context: input.Context(),
})
if err != nil {
return errorutil.NewWithTag(request.TemplateID, "could not execute pre-condition: %s", err)
@@ -218,7 +214,7 @@ func (request *Request) ExecuteWithResults(input *contextargs.Context, dynamicVa
}
}
- ctx, cancel := context.WithTimeoutCause(input.Context(), time.Duration(timeout)*time.Second, ErrCodeExecutionDeadline)
+ ctx, cancel := context.WithTimeoutCause(input.Context(), request.options.Options.GetTimeouts().CodeExecutionTimeout, ErrCodeExecutionDeadline)
defer cancel()
// Note: we use contextutil despite the fact that gozero accepts context as argument
gOutput, err := contextutil.ExecFuncWithTwoReturns(ctx, func() (*gozerotypes.Result, error) {
diff --git a/pkg/protocols/common/protocolstate/state.go b/pkg/protocols/common/protocolstate/state.go
index cf6f5234ec..b7c7796fa8 100644
--- a/pkg/protocols/common/protocolstate/state.go
+++ b/pkg/protocols/common/protocolstate/state.go
@@ -34,9 +34,7 @@ func Init(options *types.Options) error {
lfaAllowed = options.AllowLocalFileAccess
opts := fastdialer.DefaultOptions
- if options.DialerTimeout > 0 {
- opts.DialerTimeout = options.DialerTimeout
- }
+ opts.DialerTimeout = options.GetTimeouts().DialTimeout
if options.DialerKeepAlive > 0 {
opts.DialerKeepAlive = options.DialerKeepAlive
}
diff --git a/pkg/protocols/http/http.go b/pkg/protocols/http/http.go
index c20cbbfa66..fc347ce88d 100644
--- a/pkg/protocols/http/http.go
+++ b/pkg/protocols/http/http.go
@@ -336,7 +336,7 @@ func (request *Request) Compile(options *protocols.ExecutorOptions) error {
request.Raw[i] = strings.ReplaceAll(raw, "\n", "\r\n")
}
}
- request.rawhttpClient = httpclientpool.GetRawHTTP(options.Options)
+ request.rawhttpClient = httpclientpool.GetRawHTTP(options)
}
if len(request.Matchers) > 0 || len(request.Extractors) > 0 {
compiled := &request.Operators
diff --git a/pkg/protocols/http/httpclientpool/clientpool.go b/pkg/protocols/http/httpclientpool/clientpool.go
index b4ca5d40cf..2c244556ae 100644
--- a/pkg/protocols/http/httpclientpool/clientpool.go
+++ b/pkg/protocols/http/httpclientpool/clientpool.go
@@ -17,6 +17,7 @@ import (
"golang.org/x/net/publicsuffix"
"github.com/projectdiscovery/fastdialer/fastdialer/ja3/impersonate"
+ "github.com/projectdiscovery/nuclei/v3/pkg/protocols"
"github.com/projectdiscovery/nuclei/v3/pkg/protocols/common/protocolstate"
"github.com/projectdiscovery/nuclei/v3/pkg/protocols/utils"
"github.com/projectdiscovery/nuclei/v3/pkg/types"
@@ -31,28 +32,14 @@ var (
forceMaxRedirects int
normalClient *retryablehttp.Client
clientPool *mapsutil.SyncLockMap[string, *retryablehttp.Client]
- // MaxResponseHeaderTimeout is the timeout for response headers
- // to be read from the server (this prevents infinite hang started by server if any)
- // Note: this will be overridden temporarily when using @timeout request annotation
- MaxResponseHeaderTimeout = time.Duration(10) * time.Second
- // HttpTimeoutMultiplier is the multiplier for the http timeout
- HttpTimeoutMultiplier = 3
)
-// GetHttpTimeout returns the http timeout for the client
-func GetHttpTimeout(opts *types.Options) time.Duration {
- return time.Duration(opts.Timeout*HttpTimeoutMultiplier) * time.Second
-}
-
// Init initializes the clientpool implementation
func Init(options *types.Options) error {
// Don't create clients if already created in the past.
if normalClient != nil {
return nil
}
- if options.Timeout > 10 {
- MaxResponseHeaderTimeout = time.Duration(options.Timeout) * time.Second
- }
if options.ShouldFollowHTTPRedirects() {
forceMaxRedirects = options.MaxRedirects
}
@@ -143,7 +130,7 @@ func (c *Configuration) HasStandardOptions() bool {
}
// GetRawHTTP returns the rawhttp request client
-func GetRawHTTP(options *types.Options) *rawhttp.Client {
+func GetRawHTTP(options *protocols.ExecutorOptions) *rawhttp.Client {
if rawHttpClient == nil {
rawHttpOptions := rawhttp.DefaultOptions
if types.ProxyURL != "" {
@@ -153,7 +140,7 @@ func GetRawHTTP(options *types.Options) *rawhttp.Client {
} else if protocolstate.Dialer != nil {
rawHttpOptions.FastDialer = protocolstate.Dialer
}
- rawHttpOptions.Timeout = GetHttpTimeout(options)
+ rawHttpOptions.Timeout = options.Options.GetTimeouts().HttpTimeout
rawHttpClient = rawhttp.NewClient(rawHttpOptions)
}
return rawHttpClient
@@ -239,7 +226,7 @@ func wrappedGet(options *types.Options, configuration *Configuration) (*retryabl
}
// responseHeaderTimeout is max timeout for response headers to be read
- responseHeaderTimeout := MaxResponseHeaderTimeout
+ responseHeaderTimeout := options.GetTimeouts().HttpResponseHeaderTimeout
if configuration.ResponseHeaderTimeout != 0 {
responseHeaderTimeout = configuration.ResponseHeaderTimeout
}
@@ -308,7 +295,7 @@ func wrappedGet(options *types.Options, configuration *Configuration) (*retryabl
CheckRedirect: makeCheckRedirectFunc(redirectFlow, maxRedirects),
}
if !configuration.NoTimeout {
- httpclient.Timeout = GetHttpTimeout(options)
+ httpclient.Timeout = options.GetTimeouts().HttpTimeout
}
client := retryablehttp.NewWithHTTPClient(httpclient, retryableHttpOptions)
if jar != nil {
diff --git a/pkg/protocols/http/request.go b/pkg/protocols/http/request.go
index c84db691f7..0df13b8872 100644
--- a/pkg/protocols/http/request.go
+++ b/pkg/protocols/http/request.go
@@ -483,7 +483,7 @@ func (request *Request) ExecuteWithResults(input *contextargs.Context, dynamicVa
request.options.RateLimitTake()
ctx := request.newContext(input)
- ctxWithTimeout, cancel := context.WithTimeoutCause(ctx, httpclientpool.GetHttpTimeout(request.options.Options), ErrHttpEngineRequestDeadline)
+ ctxWithTimeout, cancel := context.WithTimeoutCause(ctx, request.options.Options.GetTimeouts().HttpTimeout, ErrHttpEngineRequestDeadline)
defer cancel()
generatedHttpRequest, err := generator.Make(ctxWithTimeout, input, data, payloads, dynamicValue)
diff --git a/pkg/protocols/http/request_annotations.go b/pkg/protocols/http/request_annotations.go
index c0f4b69774..fa2a5eaedd 100644
--- a/pkg/protocols/http/request_annotations.go
+++ b/pkg/protocols/http/request_annotations.go
@@ -130,6 +130,10 @@ func (r *Request) parseAnnotations(rawRequest string, request *retryablehttp.Req
if duration := reTimeoutAnnotation.FindStringSubmatch(rawRequest); len(duration) > 0 {
value := strings.TrimSpace(duration[1])
if parsed, err := time.ParseDuration(value); err == nil {
+ // to avoid dos via timeout request annotation in http template we set it to maximum of 2 minutes
+ if parsed > 2*time.Minute {
+ parsed = 2 * time.Minute
+ }
//nolint:govet // cancelled automatically by withTimeout
// global timeout is overridden by annotation by replacing context
ctx, overrides.cancelFunc = context.WithTimeoutCause(context.TODO(), parsed, ErrTimeoutAnnotationDeadline)
@@ -140,7 +144,7 @@ func (r *Request) parseAnnotations(rawRequest string, request *retryablehttp.Req
} else {
//nolint:govet // cancelled automatically by withTimeout
// global timeout is overridden by annotation by replacing context
- ctx, overrides.cancelFunc = context.WithTimeoutCause(context.TODO(), httpclientpool.GetHttpTimeout(r.options.Options), ErrRequestTimeoutDeadline)
+ ctx, overrides.cancelFunc = context.WithTimeoutCause(context.TODO(), r.options.Options.GetTimeouts().HttpTimeout, ErrRequestTimeoutDeadline)
request = request.Clone(ctx)
}
}
diff --git a/pkg/protocols/javascript/js.go b/pkg/protocols/javascript/js.go
index 71f9f53413..2e43c61cd9 100644
--- a/pkg/protocols/javascript/js.go
+++ b/pkg/protocols/javascript/js.go
@@ -65,9 +65,6 @@ type Request struct {
// Code contains code to execute for the javascript request.
Code string `yaml:"code,omitempty" json:"code,omitempty" jsonschema:"title=code to execute in javascript,description=Executes inline javascript code for the request"`
// description: |
- // Timeout in seconds is optional timeout for each javascript script execution (i.e init, pre-condition, code)
- Timeout int `yaml:"timeout,omitempty" json:"timeout,omitempty" jsonschema:"title=timeout for javascript execution,description=Timeout in seconds is optional timeout for entire javascript script execution"`
- // description: |
// StopAtFirstMatch stops processing the request at first match.
StopAtFirstMatch bool `yaml:"stop-at-first-match,omitempty" json:"stop-at-first-match,omitempty" jsonschema:"title=stop at first match,description=Stop the execution after a match is found"`
// description: |
@@ -153,9 +150,9 @@ func (request *Request) Compile(options *protocols.ExecutorOptions) error {
}
opts := &compiler.ExecuteOptions{
- Timeout: request.Timeout,
- Source: &request.Init,
- Context: context.Background(),
+ TimeoutVariants: request.options.Options.GetTimeouts(),
+ Source: &request.Init,
+ Context: context.Background(),
}
// register 'export' function to export variables from init code
// these are saved in args and are available in pre-condition and request code
@@ -345,7 +342,10 @@ func (request *Request) ExecuteWithResults(target *contextargs.Context, dynamicV
argsCopy.TemplateCtx = templateCtx.GetAll()
result, err := request.options.JsCompiler.ExecuteWithOptions(request.preConditionCompiled, argsCopy,
- &compiler.ExecuteOptions{Timeout: request.Timeout, Source: &request.PreCondition, Context: target.Context()})
+ &compiler.ExecuteOptions{
+ TimeoutVariants: requestOptions.Options.GetTimeouts(),
+ Source: &request.PreCondition, Context: target.Context(),
+ })
if err != nil {
return errorutil.NewWithTag(request.TemplateID, "could not execute pre-condition: %s", err)
}
@@ -500,7 +500,11 @@ func (request *Request) executeRequestWithPayloads(hostPort string, input *conte
}
results, err := request.options.JsCompiler.ExecuteWithOptions(request.scriptCompiled, argsCopy,
- &compiler.ExecuteOptions{Timeout: request.Timeout, Source: &request.Code, Context: input.Context()})
+ &compiler.ExecuteOptions{
+ TimeoutVariants: requestOptions.Options.GetTimeouts(),
+ Source: &request.Code,
+ Context: input.Context(),
+ })
if err != nil {
// shouldn't fail even if it returned error instead create a failure event
results = compiler.ExecuteResult{"success": false, "error": err.Error()}
diff --git a/pkg/protocols/network/request.go b/pkg/protocols/network/request.go
index 62807abf93..90390e53c3 100644
--- a/pkg/protocols/network/request.go
+++ b/pkg/protocols/network/request.go
@@ -327,7 +327,7 @@ func (request *Request) executeRequestWithPayloads(variables map[string]interfac
}
if input.Read > 0 {
- buffer, err := ConnReadNWithTimeout(conn, int64(input.Read), request.options.Options.ResponseReadTimeout)
+ buffer, err := ConnReadNWithTimeout(conn, int64(input.Read), request.options.Options.GetTimeouts().TcpReadTimeout)
if err != nil {
return errorutil.NewWithErr(err).Msgf("could not read response from connection")
}
@@ -377,7 +377,7 @@ func (request *Request) executeRequestWithPayloads(variables map[string]interfac
bufferSize = -1
}
- final, err := ConnReadNWithTimeout(conn, int64(bufferSize), request.options.Options.ResponseReadTimeout)
+ final, err := ConnReadNWithTimeout(conn, int64(bufferSize), request.options.Options.GetTimeouts().TcpReadTimeout)
if err != nil {
request.options.Output.Request(request.options.TemplatePath, address, request.Type().String(), err)
gologger.Verbose().Msgf("could not read more data from %s: %s", actualAddress, err)
diff --git a/pkg/templates/compile_test.go b/pkg/templates/compile_test.go
index 81c5f8fd51..91a858bd7e 100644
--- a/pkg/templates/compile_test.go
+++ b/pkg/templates/compile_test.go
@@ -39,15 +39,15 @@ func setup() {
progressImpl, _ := progress.NewStatsTicker(0, false, false, false, 0)
executerOpts = protocols.ExecutorOptions{
- Output: testutils.NewMockOutputWriter(options.OmitTemplate),
- Options: options,
- Progress: progressImpl,
- ProjectFile: nil,
- IssuesClient: nil,
- Browser: nil,
- Catalog: disk.NewCatalog(config.DefaultConfig.TemplatesDirectory),
- RateLimiter: ratelimit.New(context.Background(), uint(options.RateLimit), time.Second),
- Parser: templates.NewParser(),
+ Output: testutils.NewMockOutputWriter(options.OmitTemplate),
+ Options: options,
+ Progress: progressImpl,
+ ProjectFile: nil,
+ IssuesClient: nil,
+ Browser: nil,
+ Catalog: disk.NewCatalog(config.DefaultConfig.TemplatesDirectory),
+ RateLimiter: ratelimit.New(context.Background(), uint(options.RateLimit), time.Second),
+ Parser: templates.NewParser(),
}
workflowLoader, err := workflow.NewLoader(&executerOpts)
if err != nil {
diff --git a/pkg/testutils/testutils.go b/pkg/testutils/testutils.go
index 6e6f94d9eb..930787aab9 100644
--- a/pkg/testutils/testutils.go
+++ b/pkg/testutils/testutils.go
@@ -76,7 +76,6 @@ var DefaultOptions = &types.Options{
InteractionsPollDuration: 5,
GitHubTemplateRepo: []string{},
GitHubToken: "",
- ResponseReadTimeout: time.Second * 5,
}
// TemplateInfo contains info for a mock executed template.
@@ -90,17 +89,17 @@ type TemplateInfo struct {
func NewMockExecuterOptions(options *types.Options, info *TemplateInfo) *protocols.ExecutorOptions {
progressImpl, _ := progress.NewStatsTicker(0, false, false, false, 0)
executerOpts := &protocols.ExecutorOptions{
- TemplateID: info.ID,
- TemplateInfo: info.Info,
- TemplatePath: info.Path,
- Output: NewMockOutputWriter(options.OmitTemplate),
- Options: options,
- Progress: progressImpl,
- ProjectFile: nil,
- IssuesClient: nil,
- Browser: nil,
- Catalog: disk.NewCatalog(config.DefaultConfig.TemplatesDirectory),
- RateLimiter: ratelimit.New(context.Background(), uint(options.RateLimit), time.Second),
+ TemplateID: info.ID,
+ TemplateInfo: info.Info,
+ TemplatePath: info.Path,
+ Output: NewMockOutputWriter(options.OmitTemplate),
+ Options: options,
+ Progress: progressImpl,
+ ProjectFile: nil,
+ IssuesClient: nil,
+ Browser: nil,
+ Catalog: disk.NewCatalog(config.DefaultConfig.TemplatesDirectory),
+ RateLimiter: ratelimit.New(context.Background(), uint(options.RateLimit), time.Second),
}
executerOpts.CreateTemplateCtxStore()
return executerOpts
diff --git a/pkg/tmplexec/flow/flow_executor_test.go b/pkg/tmplexec/flow/flow_executor_test.go
index b47b38a2a0..cf7b1790a6 100644
--- a/pkg/tmplexec/flow/flow_executor_test.go
+++ b/pkg/tmplexec/flow/flow_executor_test.go
@@ -27,15 +27,15 @@ func setup() {
progressImpl, _ := progress.NewStatsTicker(0, false, false, false, 0)
executerOpts = protocols.ExecutorOptions{
- Output: testutils.NewMockOutputWriter(options.OmitTemplate),
- Options: options,
- Progress: progressImpl,
- ProjectFile: nil,
- IssuesClient: nil,
- Browser: nil,
- Catalog: disk.NewCatalog(config.DefaultConfig.TemplatesDirectory),
- RateLimiter: ratelimit.New(context.Background(), uint(options.RateLimit), time.Second),
- Parser: templates.NewParser(),
+ Output: testutils.NewMockOutputWriter(options.OmitTemplate),
+ Options: options,
+ Progress: progressImpl,
+ ProjectFile: nil,
+ IssuesClient: nil,
+ Browser: nil,
+ Catalog: disk.NewCatalog(config.DefaultConfig.TemplatesDirectory),
+ RateLimiter: ratelimit.New(context.Background(), uint(options.RateLimit), time.Second),
+ Parser: templates.NewParser(),
}
workflowLoader, err := workflow.NewLoader(&executerOpts)
if err != nil {
diff --git a/pkg/types/types.go b/pkg/types/types.go
index f49da194cc..48e9039d37 100644
--- a/pkg/types/types.go
+++ b/pkg/types/types.go
@@ -278,8 +278,6 @@ type Options struct {
SNI string
// InputFileMode specifies the mode of input file (jsonl, burp, openapi, swagger, etc)
InputFileMode string
- // DialerTimeout sets the timeout for network requests.
- DialerTimeout time.Duration
// DialerKeepAlive sets the keep alive duration for network requests.
DialerKeepAlive time.Duration
// Interface to use for network scan
@@ -292,8 +290,6 @@ type Options struct {
ResponseReadSize int
// ResponseSaveSize is the maximum size of response to save
ResponseSaveSize int
- // ResponseReadTimeout is response read timeout in seconds
- ResponseReadTimeout time.Duration
// Health Check
HealthCheck bool
// Time to wait between each input read operation before closing the stream
@@ -406,6 +402,79 @@ type Options struct {
HttpApiEndpoint string
// ListTemplateProfiles lists all available template profiles
ListTemplateProfiles bool
+ // timeouts contains various types of timeouts used in nuclei
+ // these timeouts are derived from dial-timeout (-timeout) with known multipliers
+ // This is internally managed and does not need to be set by user by explicitly setting
+ // this overrides the default/derived one
+ timeouts *Timeouts
+}
+
+// SetTimeouts sets the timeout variants to use for the executor
+func (opts *Options) SetTimeouts(t *Timeouts) {
+ opts.timeouts = t
+}
+
+// GetTimeouts returns the timeout variants to use for the executor
+func (eo *Options) GetTimeouts() *Timeouts {
+ if eo.timeouts != nil {
+ // redundant but apply to avoid any potential issues
+ eo.timeouts.ApplyDefaults()
+ return eo.timeouts
+ }
+ // set timeout variant value
+ eo.timeouts = NewTimeoutVariant(eo.Timeout)
+ eo.timeouts.ApplyDefaults()
+ return eo.timeouts
+}
+
+// Timeouts is a struct that contains all the timeout variants for nuclei
+// dialer timeout is used to derive other timeouts
+type Timeouts struct {
+ // DialTimeout for fastdialer (default 10s)
+ DialTimeout time.Duration
+ // Tcp(Network Protocol) Read From Connection Timeout (default 5s)
+ TcpReadTimeout time.Duration
+ // Http Response Header Timeout (default 10s)
+ // this timeout prevents infinite hangs started by server if any
+ // this is temporarily overridden when using @timeout request annotation
+ HttpResponseHeaderTimeout time.Duration
+ // HttpTimeout for http client (default -> 3 x dial-timeout = 30s)
+ HttpTimeout time.Duration
+ // JsCompilerExec timeout/deadline (default -> 2 x dial-timeout = 20s)
+ JsCompilerExecutionTimeout time.Duration
+ // CodeExecutionTimeout for code execution (default -> 3 x dial-timeout = 30s)
+ CodeExecutionTimeout time.Duration
+}
+
+// NewTimeoutVariant creates a new timeout variant with the given dial timeout in seconds
+func NewTimeoutVariant(dialTimeoutSec int) *Timeouts {
+ tv := &Timeouts{
+ DialTimeout: time.Duration(dialTimeoutSec) * time.Second,
+ }
+ tv.ApplyDefaults()
+ return tv
+}
+
+// ApplyDefaults applies default values to timeout variants when missing
+func (tv *Timeouts) ApplyDefaults() {
+ if tv.DialTimeout == 0 {
+ tv.DialTimeout = 10 * time.Second
+ }
+ if tv.TcpReadTimeout == 0 {
+ tv.TcpReadTimeout = 5 * time.Second
+ }
+ if tv.HttpResponseHeaderTimeout == 0 {
+ tv.HttpResponseHeaderTimeout = 10 * time.Second
+ }
+ if tv.HttpTimeout == 0 {
+ tv.HttpTimeout = 3 * tv.DialTimeout
+ }
+ if tv.JsCompilerExecutionTimeout == 0 {
+ tv.JsCompilerExecutionTimeout = 2 * tv.DialTimeout
+ }
+ if tv.CodeExecutionTimeout == 0 {
+ tv.CodeExecutionTimeout = 3 * tv.DialTimeout
+ }
}
// ShouldLoadResume resume file
@@ -444,7 +513,6 @@ func DefaultOptions() *Options {
MaxHostError: 30,
ResponseReadSize: 10 * unitutils.Mega,
ResponseSaveSize: unitutils.Mega,
- ResponseReadTimeout: 5 * time.Second,
}
}
From 49d85796627f37cb1471e6b1673c28fed0b9abc8 Mon Sep 17 00:00:00 2001
From: GitHub Action
Date: Mon, 15 Jul 2024 10:28:25 +0000
Subject: [PATCH 089/283] Auto Generate Syntax Docs + JSONSchema [Mon Jul 15
10:28:25 UTC 2024] :robot:
---
SYNTAX-REFERENCE.md | 13 ----------
nuclei-jsonschema.json | 5 ----
pkg/templates/templates_doc.go | 43 +++++++++++++++-------------------
3 files changed, 19 insertions(+), 42 deletions(-)
diff --git a/SYNTAX-REFERENCE.md b/SYNTAX-REFERENCE.md
index 0db5711f5a..d6449e29e4 100755
--- a/SYNTAX-REFERENCE.md
+++ b/SYNTAX-REFERENCE.md
@@ -4339,19 +4339,6 @@ Code contains code to execute for the javascript request.
-timeout
int
-
-
-
-
-Timeout in seconds is optional timeout for each javascript script execution (i.e init, pre-condition, code)
-
-
-
-
-
-
-
stop-at-first-match
bool
diff --git a/nuclei-jsonschema.json b/nuclei-jsonschema.json
index 61bdc68ff8..73ba92381e 100644
--- a/nuclei-jsonschema.json
+++ b/nuclei-jsonschema.json
@@ -941,11 +941,6 @@
"title": "code to execute in javascript",
"description": "Executes inline javascript code for the request"
},
- "timeout": {
- "type": "integer",
- "title": "timeout for javascript execution",
- "description": "Timeout in seconds is optional timeout for entire javascript script execution"
- },
"stop-at-first-match": {
"type": "boolean",
"title": "stop at first match",
diff --git a/pkg/templates/templates_doc.go b/pkg/templates/templates_doc.go
index 536b80ad7f..53864516a7 100644
--- a/pkg/templates/templates_doc.go
+++ b/pkg/templates/templates_doc.go
@@ -1929,7 +1929,7 @@ func init() {
Value: "Matched is the input which was matched upon",
},
}
- JAVASCRIPTRequestDoc.Fields = make([]encoder.Doc, 10)
+ JAVASCRIPTRequestDoc.Fields = make([]encoder.Doc, 9)
JAVASCRIPTRequestDoc.Fields[0].Name = "id"
JAVASCRIPTRequestDoc.Fields[0].Type = "string"
JAVASCRIPTRequestDoc.Fields[0].Note = ""
@@ -1955,33 +1955,28 @@ func init() {
JAVASCRIPTRequestDoc.Fields[4].Note = ""
JAVASCRIPTRequestDoc.Fields[4].Description = "Code contains code to execute for the javascript request."
JAVASCRIPTRequestDoc.Fields[4].Comments[encoder.LineComment] = "Code contains code to execute for the javascript request."
- JAVASCRIPTRequestDoc.Fields[5].Name = "timeout"
- JAVASCRIPTRequestDoc.Fields[5].Type = "int"
+ JAVASCRIPTRequestDoc.Fields[5].Name = "stop-at-first-match"
+ JAVASCRIPTRequestDoc.Fields[5].Type = "bool"
JAVASCRIPTRequestDoc.Fields[5].Note = ""
- JAVASCRIPTRequestDoc.Fields[5].Description = "Timeout in seconds is optional timeout for each javascript script execution (i.e init, pre-condition, code)"
- JAVASCRIPTRequestDoc.Fields[5].Comments[encoder.LineComment] = "Timeout in seconds is optional timeout for each javascript script execution (i.e init, pre-condition, code)"
- JAVASCRIPTRequestDoc.Fields[6].Name = "stop-at-first-match"
- JAVASCRIPTRequestDoc.Fields[6].Type = "bool"
+ JAVASCRIPTRequestDoc.Fields[5].Description = "StopAtFirstMatch stops processing the request at first match."
+ JAVASCRIPTRequestDoc.Fields[5].Comments[encoder.LineComment] = "StopAtFirstMatch stops processing the request at first match."
+ JAVASCRIPTRequestDoc.Fields[6].Name = "attack"
+ JAVASCRIPTRequestDoc.Fields[6].Type = "generators.AttackTypeHolder"
JAVASCRIPTRequestDoc.Fields[6].Note = ""
- JAVASCRIPTRequestDoc.Fields[6].Description = "StopAtFirstMatch stops processing the request at first match."
- JAVASCRIPTRequestDoc.Fields[6].Comments[encoder.LineComment] = "StopAtFirstMatch stops processing the request at first match."
- JAVASCRIPTRequestDoc.Fields[7].Name = "attack"
- JAVASCRIPTRequestDoc.Fields[7].Type = "generators.AttackTypeHolder"
+ JAVASCRIPTRequestDoc.Fields[6].Description = "Attack is the type of payload combinations to perform.\n\nSniper is each payload once, pitchfork combines multiple payload sets and clusterbomb generates\npermutations and combinations for all payloads."
+ JAVASCRIPTRequestDoc.Fields[6].Comments[encoder.LineComment] = "Attack is the type of payload combinations to perform."
+ JAVASCRIPTRequestDoc.Fields[7].Name = "threads"
+ JAVASCRIPTRequestDoc.Fields[7].Type = "int"
JAVASCRIPTRequestDoc.Fields[7].Note = ""
- JAVASCRIPTRequestDoc.Fields[7].Description = "Attack is the type of payload combinations to perform.\n\nSniper is each payload once, pitchfork combines multiple payload sets and clusterbomb generates\npermutations and combinations for all payloads."
- JAVASCRIPTRequestDoc.Fields[7].Comments[encoder.LineComment] = "Attack is the type of payload combinations to perform."
- JAVASCRIPTRequestDoc.Fields[8].Name = "threads"
- JAVASCRIPTRequestDoc.Fields[8].Type = "int"
+ JAVASCRIPTRequestDoc.Fields[7].Description = "Payload concurreny i.e threads for sending requests."
+ JAVASCRIPTRequestDoc.Fields[7].Comments[encoder.LineComment] = "Payload concurreny i.e threads for sending requests."
+
+ JAVASCRIPTRequestDoc.Fields[7].AddExample("Send requests using 10 concurrent threads", 10)
+ JAVASCRIPTRequestDoc.Fields[8].Name = "payloads"
+ JAVASCRIPTRequestDoc.Fields[8].Type = "map[string]interface{}"
JAVASCRIPTRequestDoc.Fields[8].Note = ""
- JAVASCRIPTRequestDoc.Fields[8].Description = "Payload concurreny i.e threads for sending requests."
- JAVASCRIPTRequestDoc.Fields[8].Comments[encoder.LineComment] = "Payload concurreny i.e threads for sending requests."
-
- JAVASCRIPTRequestDoc.Fields[8].AddExample("Send requests using 10 concurrent threads", 10)
- JAVASCRIPTRequestDoc.Fields[9].Name = "payloads"
- JAVASCRIPTRequestDoc.Fields[9].Type = "map[string]interface{}"
- JAVASCRIPTRequestDoc.Fields[9].Note = ""
- JAVASCRIPTRequestDoc.Fields[9].Description = "Payloads contains any payloads for the current request.\n\nPayloads support both key-values combinations where a list\nof payloads is provided, or optionally a single file can also\nbe provided as payload which will be read on run-time."
- JAVASCRIPTRequestDoc.Fields[9].Comments[encoder.LineComment] = "Payloads contains any payloads for the current request."
+ JAVASCRIPTRequestDoc.Fields[8].Description = "Payloads contains any payloads for the current request.\n\nPayloads support both key-values combinations where a list\nof payloads is provided, or optionally a single file can also\nbe provided as payload which will be read on run-time."
+ JAVASCRIPTRequestDoc.Fields[8].Comments[encoder.LineComment] = "Payloads contains any payloads for the current request."
HTTPSignatureTypeHolderDoc.Type = "http.SignatureTypeHolder"
HTTPSignatureTypeHolderDoc.Comments[encoder.LineComment] = " SignatureTypeHolder is used to hold internal type of the signature"
From 0b71d983a5fd5ddf7cf89655d037030de8a2053b Mon Sep 17 00:00:00 2001
From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com>
Date: Mon, 15 Jul 2024 11:22:16 +0000
Subject: [PATCH 090/283] chore(deps): bump github.com/projectdiscovery/dsl
from 0.1.3 to 0.1.6
Bumps [github.com/projectdiscovery/dsl](https://github.com/projectdiscovery/dsl) from 0.1.3 to 0.1.6.
- [Release notes](https://github.com/projectdiscovery/dsl/releases)
- [Commits](https://github.com/projectdiscovery/dsl/compare/v0.1.3...v0.1.6)
---
updated-dependencies:
- dependency-name: github.com/projectdiscovery/dsl
dependency-type: direct:production
update-type: version-update:semver-patch
...
Signed-off-by: dependabot[bot]
---
go.mod | 6 +++---
go.sum | 12 ++++++------
2 files changed, 9 insertions(+), 9 deletions(-)
diff --git a/go.mod b/go.mod
index 29ea6becc6..578da71d01 100644
--- a/go.mod
+++ b/go.mod
@@ -79,11 +79,11 @@ require (
github.com/mholt/archiver v3.1.1+incompatible
github.com/ory/dockertest/v3 v3.10.0
github.com/praetorian-inc/fingerprintx v1.1.9
- github.com/projectdiscovery/dsl v0.1.3
+ github.com/projectdiscovery/dsl v0.1.6
github.com/projectdiscovery/fasttemplate v0.0.2
github.com/projectdiscovery/go-smb2 v0.0.0-20240129202741-052cc450c6cb
github.com/projectdiscovery/goflags v0.1.56
- github.com/projectdiscovery/gologger v1.1.13
+ github.com/projectdiscovery/gologger v1.1.14
github.com/projectdiscovery/gostruct v0.0.2
github.com/projectdiscovery/gozero v0.0.2
github.com/projectdiscovery/httpx v1.6.3
@@ -95,7 +95,7 @@ require (
github.com/projectdiscovery/tlsx v1.1.6
github.com/projectdiscovery/uncover v1.0.8
github.com/projectdiscovery/useragent v0.0.57
- github.com/projectdiscovery/utils v0.1.6
+ github.com/projectdiscovery/utils v0.2.0
github.com/projectdiscovery/wappalyzergo v0.1.6
github.com/redis/go-redis/v9 v9.1.0
github.com/seh-msft/burpxml v1.0.1
diff --git a/go.sum b/go.sum
index 2ff05a1532..02d85e58a9 100644
--- a/go.sum
+++ b/go.sum
@@ -834,8 +834,8 @@ github.com/projectdiscovery/cdncheck v1.0.9 h1:BS15gzj9gb5AVSKqTDzPamfSgStu7nJQO
github.com/projectdiscovery/cdncheck v1.0.9/go.mod h1:18SSl1w7rMj53CGeRIZTbDoa286a6xZIxGbaiEo4Fxs=
github.com/projectdiscovery/clistats v0.0.20 h1:5jO5SLiRJ7f0nDV0ndBNmBeesbROouPooH+DGMgoWq4=
github.com/projectdiscovery/clistats v0.0.20/go.mod h1:GJ2av0KnOvK0AISQnP8hyDclYIji1LVkx2l0pwnzAu4=
-github.com/projectdiscovery/dsl v0.1.3 h1:Lxlgpx8lwq/CaBjVUmwF6KmkMDnYf9IVKlMsWCF72OE=
-github.com/projectdiscovery/dsl v0.1.3/go.mod h1:WAuQLD4KQ/94RASotE7cvk7X6kY7JQG9Fcqc3lRQfpI=
+github.com/projectdiscovery/dsl v0.1.6 h1:6TUYMwbjcYJ5OoRoawPX1SYD7wgubX/1FreC8SwFqgc=
+github.com/projectdiscovery/dsl v0.1.6/go.mod h1:1PAV9A6X+fdyMXW0SY4i0BHrO8yN2+FB2jp25+0AT04=
github.com/projectdiscovery/fastdialer v0.1.5 h1:5rNJ8I+GVzDsUqRSVLUqGTm7LsshfNYmCh20lLn1aaI=
github.com/projectdiscovery/fastdialer v0.1.5/go.mod h1:oVoMl9Y6da2timhszy0okyBvhbKDSk+zqFazGV4y6pU=
github.com/projectdiscovery/fasttemplate v0.0.2 h1:h2cISk5xDhlJEinlBQS6RRx0vOlOirB2y3Yu4PJzpiA=
@@ -846,8 +846,8 @@ github.com/projectdiscovery/go-smb2 v0.0.0-20240129202741-052cc450c6cb h1:rutG90
github.com/projectdiscovery/go-smb2 v0.0.0-20240129202741-052cc450c6cb/go.mod h1:FLjF1DmZ+POoGEiIQdWuYVwS++C/GwpX8YaCsTSm1RY=
github.com/projectdiscovery/goflags v0.1.56 h1:tJYiZN7s9Jk9DxfYOUiqOoybaIDlXyX4ZgT4B/06SyU=
github.com/projectdiscovery/goflags v0.1.56/go.mod h1:DsGF0NPpM5hGg75N3MTSvWJ4MIT7HFEAOEeWZ074+Fg=
-github.com/projectdiscovery/gologger v1.1.13 h1:7h4yCnMk+DQ/43v+A3NuA1uuI4XlJd0W0xvTZioD05E=
-github.com/projectdiscovery/gologger v1.1.13/go.mod h1:PAsVPr2aHoopCzE0BOnp2bPCv+BU/Oq81ygcP3xVwgI=
+github.com/projectdiscovery/gologger v1.1.14 h1:8SS9qDCisCFffKHzWMX+GDDArxBJ9A7DhYrESEixAYo=
+github.com/projectdiscovery/gologger v1.1.14/go.mod h1:CPk1nAZ3PqgspKBuSJR+xwY2i/Rm+P/ovahpP30EJy8=
github.com/projectdiscovery/gostruct v0.0.2 h1:s8gP8ApugGM4go1pA+sVlPDXaWqNP5BBDDSv7VEdG1M=
github.com/projectdiscovery/gostruct v0.0.2/go.mod h1:H86peL4HKwMXcQQtEa6lmC8FuD9XFt6gkNR0B/Mu5PE=
github.com/projectdiscovery/gozero v0.0.2 h1:8fJeaCjxL9tpm33uG/RsCQs6HGM/NE6eA3cjkilRQ+E=
@@ -888,8 +888,8 @@ github.com/projectdiscovery/uncover v1.0.8 h1:nE8bYJuwhqk0BEMRQRhNbjPGMy40A9gkiM
github.com/projectdiscovery/uncover v1.0.8/go.mod h1:1SwsNqjwMkJEzJQ7lQr5AHIdYd+BQlyqhO5IwIYmIAM=
github.com/projectdiscovery/useragent v0.0.57 h1:y6ML3GVJ6Sq+nhqk2gY9aD0JNaKzfpunJPjPN40eVN4=
github.com/projectdiscovery/useragent v0.0.57/go.mod h1:iChqL4rGApUsdXhAzItirEpMvYYOlruieZE+nqoKpCc=
-github.com/projectdiscovery/utils v0.1.6 h1:fzdN+oCTics3hPPOEV1FWqf/Hpenav5Ml/73aEwpTZc=
-github.com/projectdiscovery/utils v0.1.6/go.mod h1:kOnMoFfuQMY8U+nh8e32odjC00iEbiLVBcDxyAZFwCc=
+github.com/projectdiscovery/utils v0.2.0 h1:GC9FOq2mVacVYZgD2s6XW7RP6jAC/XjOKclSpLYPUUQ=
+github.com/projectdiscovery/utils v0.2.0/go.mod h1:kOnMoFfuQMY8U+nh8e32odjC00iEbiLVBcDxyAZFwCc=
github.com/projectdiscovery/wappalyzergo v0.1.6 h1:4iZDEnytsnAqjQgD5K6lmAzVx++KSQeMCDfomDtQM1M=
github.com/projectdiscovery/wappalyzergo v0.1.6/go.mod h1:wBYGKmA5BQp/NWsAy1q/jSH8N1LHWQ/LV26DuR+KzPM=
github.com/projectdiscovery/yamldoc-go v1.0.4 h1:eZoESapnMw6WAHiVgRwNqvbJEfNHEH148uthhFbG5jE=
From 3e1cd279433b845bd2a064e582926ff8b23254fb Mon Sep 17 00:00:00 2001
From: Ramana Reddy <90540245+RamanaReddy0M@users.noreply.github.com>
Date: Mon, 15 Jul 2024 04:27:12 -0700
Subject: [PATCH 091/283] Fix stop-at-first-match in headless mode fuzzing
(#5330)
---
pkg/protocols/headless/request.go | 4 ++++
1 file changed, 4 insertions(+)
diff --git a/pkg/protocols/headless/request.go b/pkg/protocols/headless/request.go
index 05bf5d3548..973dfa642a 100644
--- a/pkg/protocols/headless/request.go
+++ b/pkg/protocols/headless/request.go
@@ -215,6 +215,10 @@ func (request *Request) executeRequestWithPayloads(input *contextargs.Context, p
}
dumpResponse(event, request.options, responseBody, input.MetaInput.Input)
+ shouldStopAtFirstMatch := request.StopAtFirstMatch || request.options.StopAtFirstMatch || request.options.Options.StopAtFirstMatch
+ if shouldStopAtFirstMatch && event.HasOperatorResult() {
+ return types.ErrNoMoreRequests
+ }
return nil
}
From deb0dd2a5fab1e162ade26f16238da940b19ee94 Mon Sep 17 00:00:00 2001
From: boyhack <34109680@qq.com>
Date: Mon, 15 Jul 2024 19:32:14 +0800
Subject: [PATCH 092/283] fix(engine): Resolve issue with ExecuteWithResults
function not returning expected results (#5376)
fix(engine): Resolve issue with ExecuteWithResults function not returning expected results
When attempting to use the `ExecuteWithResults` function, users were finding that the function was not returning the expected results. This fix addresses the root cause of this problem.
---
pkg/core/executors.go | 6 +++---
1 file changed, 3 insertions(+), 3 deletions(-)
diff --git a/pkg/core/executors.go b/pkg/core/executors.go
index 0e91c3883b..8447efed13 100644
--- a/pkg/core/executors.go
+++ b/pkg/core/executors.go
@@ -27,7 +27,7 @@ func (e *Engine) executeAllSelfContained(ctx context.Context, alltemplates []*te
var match bool
ctx := scan.NewScanContext(ctx, contextargs.New(ctx))
if e.Callback != nil {
- if results, err := template.Executer.ExecuteWithResults(ctx); err != nil {
+ if results, err := template.Executer.ExecuteWithResults(ctx); err == nil {
for _, result := range results {
e.Callback(result)
}
@@ -129,7 +129,7 @@ func (e *Engine) executeTemplateWithTargets(ctx context.Context, template *templ
match = e.executeWorkflow(ctx, template.CompiledWorkflow)
default:
if e.Callback != nil {
- if results, err := template.Executer.ExecuteWithResults(ctx); err != nil {
+ if results, err := template.Executer.ExecuteWithResults(ctx); err == nil {
for _, result := range results {
e.Callback(result)
}
@@ -194,7 +194,7 @@ func (e *Engine) executeTemplatesOnTarget(ctx context.Context, alltemplates []*t
match = e.executeWorkflow(ctx, template.CompiledWorkflow)
default:
if e.Callback != nil {
- if results, err := template.Executer.ExecuteWithResults(ctx); err != nil {
+ if results, err := template.Executer.ExecuteWithResults(ctx); err == nil {
for _, result := range results {
e.Callback(result)
}
From ceea533681abcb89a1b3757a56a28bf4fa56bc4c Mon Sep 17 00:00:00 2001
From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com>
Date: Mon, 15 Jul 2024 11:48:13 +0000
Subject: [PATCH 093/283] chore(deps): bump github.com/projectdiscovery/goflags
Bumps [github.com/projectdiscovery/goflags](https://github.com/projectdiscovery/goflags) from 0.1.56 to 0.1.59.
- [Release notes](https://github.com/projectdiscovery/goflags/releases)
- [Commits](https://github.com/projectdiscovery/goflags/compare/v0.1.56...v0.1.59)
---
updated-dependencies:
- dependency-name: github.com/projectdiscovery/goflags
dependency-type: direct:production
update-type: version-update:semver-patch
...
Signed-off-by: dependabot[bot]
---
go.mod | 2 +-
go.sum | 4 ++--
2 files changed, 3 insertions(+), 3 deletions(-)
diff --git a/go.mod b/go.mod
index 578da71d01..25c924037a 100644
--- a/go.mod
+++ b/go.mod
@@ -82,7 +82,7 @@ require (
github.com/projectdiscovery/dsl v0.1.6
github.com/projectdiscovery/fasttemplate v0.0.2
github.com/projectdiscovery/go-smb2 v0.0.0-20240129202741-052cc450c6cb
- github.com/projectdiscovery/goflags v0.1.56
+ github.com/projectdiscovery/goflags v0.1.59
github.com/projectdiscovery/gologger v1.1.14
github.com/projectdiscovery/gostruct v0.0.2
github.com/projectdiscovery/gozero v0.0.2
diff --git a/go.sum b/go.sum
index 02d85e58a9..859dee0c5b 100644
--- a/go.sum
+++ b/go.sum
@@ -844,8 +844,8 @@ github.com/projectdiscovery/freeport v0.0.5 h1:jnd3Oqsl4S8n0KuFkE5Hm8WGDP24ITBvm
github.com/projectdiscovery/freeport v0.0.5/go.mod h1:PY0bxSJ34HVy67LHIeF3uIutiCSDwOqKD8ruBkdiCwE=
github.com/projectdiscovery/go-smb2 v0.0.0-20240129202741-052cc450c6cb h1:rutG906Drtbpz4DwU5mhGIeOhRcktDH4cGQitGUMAsg=
github.com/projectdiscovery/go-smb2 v0.0.0-20240129202741-052cc450c6cb/go.mod h1:FLjF1DmZ+POoGEiIQdWuYVwS++C/GwpX8YaCsTSm1RY=
-github.com/projectdiscovery/goflags v0.1.56 h1:tJYiZN7s9Jk9DxfYOUiqOoybaIDlXyX4ZgT4B/06SyU=
-github.com/projectdiscovery/goflags v0.1.56/go.mod h1:DsGF0NPpM5hGg75N3MTSvWJ4MIT7HFEAOEeWZ074+Fg=
+github.com/projectdiscovery/goflags v0.1.59 h1:rFScpC57L0Ln0dJmBtstyxHUH7K9H4UmHACXcSkdqqk=
+github.com/projectdiscovery/goflags v0.1.59/go.mod h1:dj67QGp/D81WRLtzU0HsWR20zgoGZ0cnk3Wbt9xJcuo=
github.com/projectdiscovery/gologger v1.1.14 h1:8SS9qDCisCFffKHzWMX+GDDArxBJ9A7DhYrESEixAYo=
github.com/projectdiscovery/gologger v1.1.14/go.mod h1:CPk1nAZ3PqgspKBuSJR+xwY2i/Rm+P/ovahpP30EJy8=
github.com/projectdiscovery/gostruct v0.0.2 h1:s8gP8ApugGM4go1pA+sVlPDXaWqNP5BBDDSv7VEdG1M=
From 455a900a961b671352b2293f640c4307fac0a52f Mon Sep 17 00:00:00 2001
From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com>
Date: Mon, 15 Jul 2024 11:48:27 +0000
Subject: [PATCH 094/283] chore(deps): bump
github.com/projectdiscovery/cdncheck
Bumps [github.com/projectdiscovery/cdncheck](https://github.com/projectdiscovery/cdncheck) from 1.0.9 to 1.1.0.
- [Release notes](https://github.com/projectdiscovery/cdncheck/releases)
- [Changelog](https://github.com/projectdiscovery/cdncheck/blob/main/.goreleaser.yaml)
- [Commits](https://github.com/projectdiscovery/cdncheck/compare/v1.0.9...v1.1.0)
---
updated-dependencies:
- dependency-name: github.com/projectdiscovery/cdncheck
dependency-type: indirect
update-type: version-update:semver-minor
...
Signed-off-by: dependabot[bot]
---
go.mod | 3 +--
go.sum | 4 ++--
2 files changed, 3 insertions(+), 4 deletions(-)
diff --git a/go.mod b/go.mod
index 578da71d01..4cbc641f03 100644
--- a/go.mod
+++ b/go.mod
@@ -201,7 +201,7 @@ require (
github.com/pjbgf/sha1cd v0.3.0 // indirect
github.com/pkg/browser v0.0.0-20240102092130-5ac0b6a4141c // indirect
github.com/projectdiscovery/asnmap v1.1.1 // indirect
- github.com/projectdiscovery/cdncheck v1.0.9 // indirect
+ github.com/projectdiscovery/cdncheck v1.1.0 // indirect
github.com/projectdiscovery/freeport v0.0.5 // indirect
github.com/projectdiscovery/ldapserver v1.0.2-0.20240219154113-dcc758ebc0cb // indirect
github.com/projectdiscovery/machineid v0.0.0-20240226150047-2e2c51e35983 // indirect
@@ -293,7 +293,6 @@ require (
github.com/trivago/tgo v1.0.7
github.com/ulikunitz/xz v0.5.12 // indirect
github.com/valyala/bytebufferpool v1.0.0 // indirect
- github.com/yl2chen/cidranger v1.0.2 // indirect
github.com/ysmood/goob v0.4.0 // indirect
github.com/ysmood/gson v0.7.3 // indirect
github.com/ysmood/leakless v0.8.0 // indirect
diff --git a/go.sum b/go.sum
index 02d85e58a9..35c8639ef7 100644
--- a/go.sum
+++ b/go.sum
@@ -830,8 +830,8 @@ github.com/projectdiscovery/asnmap v1.1.1 h1:ImJiKIaACOT7HPx4Pabb5dksolzaFYsD1kI
github.com/projectdiscovery/asnmap v1.1.1/go.mod h1:QT7jt9nQanj+Ucjr9BqGr1Q2veCCKSAVyUzLXfEcQ60=
github.com/projectdiscovery/blackrock v0.0.1 h1:lHQqhaaEFjgf5WkuItbpeCZv2DUIE45k0VbGJyft6LQ=
github.com/projectdiscovery/blackrock v0.0.1/go.mod h1:ANUtjDfaVrqB453bzToU+YB4cUbvBRpLvEwoWIwlTss=
-github.com/projectdiscovery/cdncheck v1.0.9 h1:BS15gzj9gb5AVSKqTDzPamfSgStu7nJQOocUvrssFlg=
-github.com/projectdiscovery/cdncheck v1.0.9/go.mod h1:18SSl1w7rMj53CGeRIZTbDoa286a6xZIxGbaiEo4Fxs=
+github.com/projectdiscovery/cdncheck v1.1.0 h1:qDITidmJsejzpk3rMkauCh6sjI2GH9hW/snk0cQ3kXE=
+github.com/projectdiscovery/cdncheck v1.1.0/go.mod h1:sZ8U4MjHSsyaTVjBbYWHT1cwUVvUYwDX1W+WvWRicIc=
github.com/projectdiscovery/clistats v0.0.20 h1:5jO5SLiRJ7f0nDV0ndBNmBeesbROouPooH+DGMgoWq4=
github.com/projectdiscovery/clistats v0.0.20/go.mod h1:GJ2av0KnOvK0AISQnP8hyDclYIji1LVkx2l0pwnzAu4=
github.com/projectdiscovery/dsl v0.1.6 h1:6TUYMwbjcYJ5OoRoawPX1SYD7wgubX/1FreC8SwFqgc=
From 2e78b50f8049ff844edaf33a09e23f1b078a7f03 Mon Sep 17 00:00:00 2001
From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com>
Date: Mon, 15 Jul 2024 11:48:28 +0000
Subject: [PATCH 095/283] chore(deps): bump github.com/projectdiscovery/rawhttp
Bumps [github.com/projectdiscovery/rawhttp](https://github.com/projectdiscovery/rawhttp) from 0.1.51 to 0.1.57.
- [Release notes](https://github.com/projectdiscovery/rawhttp/releases)
- [Commits](https://github.com/projectdiscovery/rawhttp/compare/v0.1.51...v0.1.57)
---
updated-dependencies:
- dependency-name: github.com/projectdiscovery/rawhttp
dependency-type: direct:production
update-type: version-update:semver-patch
...
Signed-off-by: dependabot[bot]
---
go.mod | 8 ++++----
go.sum | 16 ++++++++--------
2 files changed, 12 insertions(+), 12 deletions(-)
diff --git a/go.mod b/go.mod
index 578da71d01..018abb64f2 100644
--- a/go.mod
+++ b/go.mod
@@ -20,12 +20,12 @@ require (
github.com/olekukonko/tablewriter v0.0.5
github.com/pkg/errors v0.9.1
github.com/projectdiscovery/clistats v0.0.20
- github.com/projectdiscovery/fastdialer v0.1.5
- github.com/projectdiscovery/hmap v0.0.48
+ github.com/projectdiscovery/fastdialer v0.1.6
+ github.com/projectdiscovery/hmap v0.0.49
github.com/projectdiscovery/interactsh v1.2.0
- github.com/projectdiscovery/rawhttp v0.1.51
+ github.com/projectdiscovery/rawhttp v0.1.57
github.com/projectdiscovery/retryabledns v1.0.65
- github.com/projectdiscovery/retryablehttp-go v1.0.67
+ github.com/projectdiscovery/retryablehttp-go v1.0.68
github.com/projectdiscovery/yamldoc-go v1.0.4
github.com/remeh/sizedwaitgroup v1.0.0
github.com/rs/xid v1.5.0
diff --git a/go.sum b/go.sum
index 02d85e58a9..260c57428f 100644
--- a/go.sum
+++ b/go.sum
@@ -836,8 +836,8 @@ github.com/projectdiscovery/clistats v0.0.20 h1:5jO5SLiRJ7f0nDV0ndBNmBeesbROouPo
github.com/projectdiscovery/clistats v0.0.20/go.mod h1:GJ2av0KnOvK0AISQnP8hyDclYIji1LVkx2l0pwnzAu4=
github.com/projectdiscovery/dsl v0.1.6 h1:6TUYMwbjcYJ5OoRoawPX1SYD7wgubX/1FreC8SwFqgc=
github.com/projectdiscovery/dsl v0.1.6/go.mod h1:1PAV9A6X+fdyMXW0SY4i0BHrO8yN2+FB2jp25+0AT04=
-github.com/projectdiscovery/fastdialer v0.1.5 h1:5rNJ8I+GVzDsUqRSVLUqGTm7LsshfNYmCh20lLn1aaI=
-github.com/projectdiscovery/fastdialer v0.1.5/go.mod h1:oVoMl9Y6da2timhszy0okyBvhbKDSk+zqFazGV4y6pU=
+github.com/projectdiscovery/fastdialer v0.1.6 h1:dtdE7WjaydpndKPcUYWms5iOhBSC0nvBAHyZd6I4pdQ=
+github.com/projectdiscovery/fastdialer v0.1.6/go.mod h1:a5XOjvaO4b+ddO/YLoM4nvK3CPPviW6HmoAd+AoszMw=
github.com/projectdiscovery/fasttemplate v0.0.2 h1:h2cISk5xDhlJEinlBQS6RRx0vOlOirB2y3Yu4PJzpiA=
github.com/projectdiscovery/fasttemplate v0.0.2/go.mod h1:XYWWVMxnItd+r0GbjA1GCsUopMw1/XusuQxdyAIHMCw=
github.com/projectdiscovery/freeport v0.0.5 h1:jnd3Oqsl4S8n0KuFkE5Hm8WGDP24ITBvmyw5pFTHS8Q=
@@ -852,8 +852,8 @@ github.com/projectdiscovery/gostruct v0.0.2 h1:s8gP8ApugGM4go1pA+sVlPDXaWqNP5BBD
github.com/projectdiscovery/gostruct v0.0.2/go.mod h1:H86peL4HKwMXcQQtEa6lmC8FuD9XFt6gkNR0B/Mu5PE=
github.com/projectdiscovery/gozero v0.0.2 h1:8fJeaCjxL9tpm33uG/RsCQs6HGM/NE6eA3cjkilRQ+E=
github.com/projectdiscovery/gozero v0.0.2/go.mod h1:d8bZvDWW07LWNYWrwjZ4OO1I0cpkfqaysyDfSs9ibK8=
-github.com/projectdiscovery/hmap v0.0.48 h1:pbxoDcwZWSdaqZ3QgaG5Aw1G1qEFN2zeQco6eWeROAo=
-github.com/projectdiscovery/hmap v0.0.48/go.mod h1:KMSbPJoIgI+9PkG6alZ0INEMRiIGnND128RUjxiY+8I=
+github.com/projectdiscovery/hmap v0.0.49 h1:QNW98JDqd0nmaIFRMbn0sLocRIGXAud1liGip4iPdvc=
+github.com/projectdiscovery/hmap v0.0.49/go.mod h1:BA4cyYTrWKMUa78815R6p3tuGiYu2MJHYnjwXmefdbU=
github.com/projectdiscovery/httpx v1.6.3 h1:TSu3zEKOHGFfMsGeB0lZEFRl6Tq4nPuIgKDXi9NnYpg=
github.com/projectdiscovery/httpx v1.6.3/go.mod h1:9FmsmkT71aTa4WnOyBIPzB8I9c0QAJ9PSVUqs2NkHRQ=
github.com/projectdiscovery/interactsh v1.2.0 h1:Al6jHiR+Usl9egYJDLJaWNHOcH8Rugk8gWMasc8Cmw8=
@@ -870,14 +870,14 @@ github.com/projectdiscovery/networkpolicy v0.0.9 h1:IrlDoYZagNNO8y+7iZeHT8k5izE+
github.com/projectdiscovery/networkpolicy v0.0.9/go.mod h1:XFJ2Lnv8BE/ziQCFjBHMsH1w6VmkPiQtk+NlBpdMU7M=
github.com/projectdiscovery/ratelimit v0.0.45 h1:h28oF+hJ0CHcdBZozT1Go7ppWmzTxSXDKNNh2G1Ot9Q=
github.com/projectdiscovery/ratelimit v0.0.45/go.mod h1:1vSJUseDS7SjNwIBi9wNRcgsMKNTLxy/GfdlLFVbgI4=
-github.com/projectdiscovery/rawhttp v0.1.51 h1:T4sskU2fsX9zNR7nyae0tD/0fliIs73KwBzyoJwBs10=
-github.com/projectdiscovery/rawhttp v0.1.51/go.mod h1:ejBaw3dCfJSfbtFv0ib6YfFSjQzt7ZdRVpQWJASxIUs=
+github.com/projectdiscovery/rawhttp v0.1.57 h1:2vCT2i1NSZbTBH+uUBrxOJjxDPKgIl2q6BGtQjs/Hko=
+github.com/projectdiscovery/rawhttp v0.1.57/go.mod h1:qtthyaU0k8eqcEdza1R/fTqwyxSK4BZ511ThxgkiQtE=
github.com/projectdiscovery/rdap v0.9.1-0.20221108103045-9865884d1917 h1:m03X4gBVSorSzvmm0bFa7gDV4QNSOWPL/fgZ4kTXBxk=
github.com/projectdiscovery/rdap v0.9.1-0.20221108103045-9865884d1917/go.mod h1:JxXtZC9e195awe7EynrcnBJmFoad/BNDzW9mzFkK8Sg=
github.com/projectdiscovery/retryabledns v1.0.65 h1:wOtMuVkrcrGkzaT8Jixb3BtUcJODXLRz+z9EjftmFsQ=
github.com/projectdiscovery/retryabledns v1.0.65/go.mod h1:lf3xRdFhGh55fWVlzK1oAaDPrsKliqH3gGvsFNAOkCk=
-github.com/projectdiscovery/retryablehttp-go v1.0.67 h1:3jCiREsQEWJN2fG3D+i46zJNkq5+GtrLGf+Q77jGgrE=
-github.com/projectdiscovery/retryablehttp-go v1.0.67/go.mod h1:LXTRQdTOOmMLJDbw5R9fyYgQKLjwS/l5tVt2wP09FBI=
+github.com/projectdiscovery/retryablehttp-go v1.0.68 h1:MN/kfccDnebmp1kNcCnGlGuCfvet6y5eN1Y1hIKBB1I=
+github.com/projectdiscovery/retryablehttp-go v1.0.68/go.mod h1:F3QNCoJExPFT59AACrVgYVhNiDW5Fp8s5HbXc+8CaaU=
github.com/projectdiscovery/sarif v0.0.1 h1:C2Tyj0SGOKbCLgHrx83vaE6YkzXEVrMXYRGLkKCr/us=
github.com/projectdiscovery/sarif v0.0.1/go.mod h1:cEYlDu8amcPf6b9dSakcz2nNnJsoz4aR6peERwV+wuQ=
github.com/projectdiscovery/stringsutil v0.0.2 h1:uzmw3IVLJSMW1kEg8eCStG/cGbYYZAja8BH3LqqJXMA=
From 6cbd73f780bc322420f6bfe481a68b68c8c34a5f Mon Sep 17 00:00:00 2001
From: Tarun Koyalwar <45962551+tarunKoyalwar@users.noreply.github.com>
Date: Mon, 15 Jul 2024 18:42:22 +0530
Subject: [PATCH 096/283] feat: improve ldap output with custom type: (#5387)
* feat: improve ldap output with custom type:
* js bindings update
* lint fix
---
pkg/js/compiler/pool.go | 2 +-
pkg/js/generated/go/libldap/ldap.go | 10 +-
pkg/js/generated/ts/kerberos.ts | 96 +++++-----
pkg/js/generated/ts/ldap.ts | 276 +++++++++++++++++++++-------
pkg/js/generated/ts/rdp.ts | 4 +-
pkg/js/generated/ts/smb.ts | 24 +--
pkg/js/generated/ts/ssh.ts | 42 ++---
pkg/js/libs/ldap/adenum.go | 70 +++----
pkg/js/libs/ldap/ldap.go | 33 +---
pkg/js/libs/ldap/utils.go | 181 ++++++++++++++++++
10 files changed, 515 insertions(+), 223 deletions(-)
diff --git a/pkg/js/compiler/pool.go b/pkg/js/compiler/pool.go
index 17d9f746f5..3407d97369 100644
--- a/pkg/js/compiler/pool.go
+++ b/pkg/js/compiler/pool.go
@@ -241,5 +241,5 @@ func stringify(gojaValue goja.Value, runtime *goja.Runtime) string {
}
}
// for everything else stringify
- return fmt.Sprintf("%v", value)
+ return fmt.Sprintf("%+v", value)
}
diff --git a/pkg/js/generated/go/libldap/ldap.go b/pkg/js/generated/go/libldap/ldap.go
index 8d0c2702e4..978ded0c09 100644
--- a/pkg/js/generated/go/libldap/ldap.go
+++ b/pkg/js/generated/go/libldap/ldap.go
@@ -53,10 +53,12 @@ func init() {
"FilterWorkstationTrustAccount": lib_ldap.FilterWorkstationTrustAccount,
// Objects / Classes
- "ADObject": gojs.GetClassConstructor[lib_ldap.ADObject](&lib_ldap.ADObject{}),
- "Client": lib_ldap.NewClient,
- "Config": gojs.GetClassConstructor[lib_ldap.Config](&lib_ldap.Config{}),
- "Metadata": gojs.GetClassConstructor[lib_ldap.Metadata](&lib_ldap.Metadata{}),
+ "Client": lib_ldap.NewClient,
+ "Config": gojs.GetClassConstructor[lib_ldap.Config](&lib_ldap.Config{}),
+ "LdapAttributes": gojs.GetClassConstructor[lib_ldap.LdapAttributes](&lib_ldap.LdapAttributes{}),
+ "LdapEntry": gojs.GetClassConstructor[lib_ldap.LdapEntry](&lib_ldap.LdapEntry{}),
+ "Metadata": gojs.GetClassConstructor[lib_ldap.Metadata](&lib_ldap.Metadata{}),
+ "SearchResult": gojs.GetClassConstructor[lib_ldap.SearchResult](&lib_ldap.SearchResult{}),
},
).Register()
}
diff --git a/pkg/js/generated/ts/kerberos.ts b/pkg/js/generated/ts/kerberos.ts
index 8f9ce98f99..d0af2d0e7e 100755
--- a/pkg/js/generated/ts/kerberos.ts
+++ b/pkg/js/generated/ts/kerberos.ts
@@ -188,9 +188,9 @@ export class Config {
*/
export interface AuthorizationDataEntry {
- ADType?: number,
-
ADData?: Uint8Array,
+
+ ADType?: number,
}
@@ -200,9 +200,9 @@ export interface AuthorizationDataEntry {
*/
export interface BitString {
- BitLength?: number,
-
Bytes?: Uint8Array,
+
+ BitLength?: number,
}
@@ -212,9 +212,9 @@ export interface BitString {
*/
export interface BitString {
- BitLength?: number,
-
Bytes?: Uint8Array,
+
+ BitLength?: number,
}
@@ -236,19 +236,15 @@ export interface Config {
*/
export interface EncTicketPart {
- AuthTime?: Date,
-
- StartTime?: Date,
-
EndTime?: Date,
RenewTill?: Date,
CRealm?: string,
- CAddr?: HostAddress,
+ AuthTime?: Date,
- AuthorizationData?: AuthorizationDataEntry,
+ StartTime?: Date,
Flags?: BitString,
@@ -257,6 +253,10 @@ export interface EncTicketPart {
CName?: PrincipalName,
Transited?: TransitedEncoding,
+
+ CAddr?: HostAddress,
+
+ AuthorizationData?: AuthorizationDataEntry,
}
@@ -266,11 +266,11 @@ export interface EncTicketPart {
*/
export interface EncryptedData {
+ EType?: number,
+
KVNO?: number,
Cipher?: Uint8Array,
-
- EType?: number,
}
@@ -318,41 +318,33 @@ export interface HostAddress {
*/
export interface LibDefaults {
- NoAddresses?: boolean,
+ CCacheType?: number,
- RealmTryDomains?: number,
+ K5LoginAuthoritative?: boolean,
- DNSLookupKDC?: boolean,
+ Proxiable?: boolean,
- DefaultRealm?: string,
+ RDNS?: boolean,
- SafeChecksumType?: number,
+ K5LoginDirectory?: string,
- VerifyAPReqNofail?: boolean,
+ KDCTimeSync?: number,
- AllowWeakCrypto?: boolean,
+ VerifyAPReqNofail?: boolean,
DefaultTGSEnctypes?: string[],
- DefaultTktEnctypeIDs?: number[],
-
- IgnoreAcceptorHostname?: boolean,
+ DefaultTGSEnctypeIDs?: number[],
- K5LoginAuthoritative?: boolean,
+ DNSCanonicalizeHostname?: boolean,
- PermittedEnctypes?: string[],
+ Forwardable?: boolean,
/**
* time in nanoseconds
*/
- Clockskew?: number,
-
- DNSCanonicalizeHostname?: boolean,
-
- Proxiable?: boolean,
-
- RDNS?: boolean,
+ RenewLifetime?: number,
/**
* time in nanoseconds
@@ -362,37 +354,45 @@ export interface LibDefaults {
DefaultClientKeytabName?: string,
+ DefaultTktEnctypeIDs?: number[],
+
+ DNSLookupRealm?: boolean,
+
+ ExtraAddresses?: Uint8Array,
+
+ DefaultRealm?: string,
+
+ NoAddresses?: boolean,
+
+ PreferredPreauthTypes?: number[],
+
PermittedEnctypeIDs?: number[],
- UDPPreferenceLimit?: number,
+ RealmTryDomains?: number,
- DefaultTGSEnctypeIDs?: number[],
+ DefaultKeytabName?: string,
DefaultTktEnctypes?: string[],
- CCacheType?: number,
-
- DNSLookupRealm?: boolean,
+ DNSLookupKDC?: boolean,
- ExtraAddresses?: Uint8Array,
+ IgnoreAcceptorHostname?: boolean,
- PreferredPreauthTypes?: number[],
+ AllowWeakCrypto?: boolean,
Canonicalize?: boolean,
- Forwardable?: boolean,
-
- K5LoginDirectory?: string,
+ SafeChecksumType?: number,
- KDCTimeSync?: number,
+ UDPPreferenceLimit?: number,
/**
* time in nanoseconds
*/
- RenewLifetime?: number,
+ Clockskew?: number,
- DefaultKeytabName?: string,
+ PermittedEnctypes?: string[],
KDCDefaultOptions?: BitString,
}
@@ -416,6 +416,8 @@ export interface PrincipalName {
*/
export interface Realm {
+ Realm?: string,
+
AdminServer?: string[],
DefaultDomain?: string,
@@ -425,8 +427,6 @@ export interface Realm {
KPasswdServer?: string[],
MasterKDC?: string[],
-
- Realm?: string,
}
diff --git a/pkg/js/generated/ts/ldap.ts b/pkg/js/generated/ts/ldap.ts
index daca4f86b9..02a10075b9 100755
--- a/pkg/js/generated/ts/ldap.ts
+++ b/pkg/js/generated/ts/ldap.ts
@@ -209,8 +209,8 @@ export class Client {
* log(to_json(users));
* ```
*/
- public FindADObjects(filter: string): ADObject[] {
- return [];
+ public FindADObjects(filter: string): SearchResult | null {
+ return null;
}
@@ -225,8 +225,8 @@ export class Client {
* log(to_json(users));
* ```
*/
- public GetADUsers(): ADObject[] {
- return [];
+ public GetADUsers(): SearchResult | null {
+ return null;
}
@@ -241,8 +241,8 @@ export class Client {
* log(to_json(users));
* ```
*/
- public GetADActiveUsers(): ADObject[] {
- return [];
+ public GetADActiveUsers(): SearchResult | null {
+ return null;
}
@@ -257,8 +257,8 @@ export class Client {
* log(to_json(users));
* ```
*/
- public GetADUserWithNeverExpiringPasswords(): ADObject[] {
- return [];
+ public GetADUserWithNeverExpiringPasswords(): SearchResult | null {
+ return null;
}
@@ -273,8 +273,8 @@ export class Client {
* log(to_json(users));
* ```
*/
- public GetADUserTrustedForDelegation(): ADObject[] {
- return [];
+ public GetADUserTrustedForDelegation(): SearchResult | null {
+ return null;
}
@@ -289,8 +289,8 @@ export class Client {
* log(to_json(users));
* ```
*/
- public GetADUserWithPasswordNotRequired(): ADObject[] {
- return [];
+ public GetADUserWithPasswordNotRequired(): SearchResult | null {
+ return null;
}
@@ -305,8 +305,8 @@ export class Client {
* log(to_json(groups));
* ```
*/
- public GetADGroups(): ADObject[] {
- return [];
+ public GetADGroups(): SearchResult | null {
+ return null;
}
@@ -321,8 +321,8 @@ export class Client {
* log(to_json(dcs));
* ```
*/
- public GetADDCList(): ADObject[] {
- return [];
+ public GetADDCList(): SearchResult | null {
+ return null;
}
@@ -337,8 +337,8 @@ export class Client {
* log(to_json(admins));
* ```
*/
- public GetADAdmins(): ADObject[] {
- return [];
+ public GetADAdmins(): SearchResult | null {
+ return null;
}
@@ -353,8 +353,8 @@ export class Client {
* log(to_json(kerberoastable));
* ```
*/
- public GetADUserKerberoastable(): ADObject[] {
- return [];
+ public GetADUserKerberoastable(): SearchResult | null {
+ return null;
}
@@ -369,8 +369,8 @@ export class Client {
* log(to_json(AsRepRoastable));
* ```
*/
- public GetADUserAsRepRoastable(): ADObject[] {
- return [];
+ public GetADUserAsRepRoastable(): SearchResult | null {
+ return null;
}
@@ -428,8 +428,8 @@ export class Client {
* const results = client.Search('(objectClass=*)', 'cn', 'mail');
* ```
*/
- public Search(filter: string, attributes: any): Record[] {
- return [];
+ public Search(filter: string, attributes: any): SearchResult | null {
+ return null;
}
@@ -481,33 +481,6 @@ export class Client {
-/**
- * ADObject represents an Active Directory object
- * @example
- * ```javascript
- * const ldap = require('nuclei/ldap');
- * const client = new ldap.Client('ldap://ldap.example.com', 'acme.com');
- * const users = client.GetADUsers();
- * log(to_json(users));
- * ```
- */
-export interface ADObject {
-
- DistinguishedName?: string,
-
- SAMAccountName?: string,
-
- PWDLastSet?: string,
-
- LastLogon?: string,
-
- MemberOf?: string[],
-
- ServicePrincipalName?: string[],
-}
-
-
-
/**
* Config is extra configuration for the ldap client
* @example
@@ -535,27 +508,186 @@ export interface Config {
/**
- * Entry Interface
+ * LdapAttributes represents all LDAP attributes of a particular
+ * ldap entry
*/
-export interface Entry {
+export interface LdapAttributes {
- DN?: string,
+ /**
+ * CurrentTime contains current time
+ */
+
+ CurrentTime?: string[],
+
+ /**
+ * SubschemaSubentry contains subschema subentry
+ */
+
+ SubschemaSubentry?: string[],
+
+ /**
+ * DsServiceName contains ds service name
+ */
+
+ DsServiceName?: string[],
+
+ /**
+ * NamingContexts contains naming contexts
+ */
+
+ NamingContexts?: string[],
+
+ /**
+ * DefaultNamingContext contains default naming context
+ */
+
+ DefaultNamingContext?: string[],
+
+ /**
+ * SchemaNamingContext contains schema naming context
+ */
+
+ SchemaNamingContext?: string[],
+
+ /**
+ * ConfigurationNamingContext contains configuration naming context
+ */
+
+ ConfigurationNamingContext?: string[],
+
+ /**
+ * RootDomainNamingContext contains root domain naming context
+ */
+
+ RootDomainNamingContext?: string[],
+
+ /**
+ * SupportedLDAPVersion contains supported LDAP version
+ */
+
+ SupportedLDAPVersion?: string[],
+
+ /**
+ * HighestCommittedUSN contains highest committed USN
+ */
+
+ HighestCommittedUSN?: string[],
+
+ /**
+ * SupportedSASLMechanisms contains supported SASL mechanisms
+ */
+
+ SupportedSASLMechanisms?: string[],
+
+ /**
+ * DnsHostName contains DNS host name
+ */
+
+ DnsHostName?: string[],
- Attributes?: EntryAttribute,
+ /**
+ * LdapServiceName contains LDAP service name
+ */
+
+ LdapServiceName?: string[],
+
+ /**
+ * ServerName contains server name
+ */
+
+ ServerName?: string[],
+
+ /**
+ * IsSynchronized contains is synchronized
+ */
+
+ IsSynchronized?: string[],
+
+ /**
+ * IsGlobalCatalogReady contains is global catalog ready
+ */
+
+ IsGlobalCatalogReady?: string[],
+
+ /**
+ * DomainFunctionality contains domain functionality
+ */
+
+ DomainFunctionality?: string[],
+
+ /**
+ * ForestFunctionality contains forest functionality
+ */
+
+ ForestFunctionality?: string[],
+
+ /**
+ * DomainControllerFunctionality contains domain controller functionality
+ */
+
+ DomainControllerFunctionality?: string[],
+
+ /**
+ * DistinguishedName contains the distinguished name
+ */
+
+ DistinguishedName?: string[],
+
+ /**
+ * SAMAccountName contains the SAM account name
+ */
+
+ SAMAccountName?: string[],
+
+ /**
+ * PWDLastSet contains the password last set time
+ */
+
+ PWDLastSet?: string[],
+
+ /**
+ * LastLogon contains the last logon time
+ */
+
+ LastLogon?: string[],
+
+ /**
+ * MemberOf contains the groups the entry is a member of
+ */
+
+ MemberOf?: string[],
+
+ /**
+ * ServicePrincipalName contains the service principal names
+ */
+
+ ServicePrincipalName?: string[],
+
+ /**
+ * Extra contains other extra fields which might be present
+ */
+
+ Extra?: Record,
}
/**
- * EntryAttribute Interface
+ * LdapEntry represents a single LDAP entry
*/
-export interface EntryAttribute {
+export interface LdapEntry {
- Name?: string,
+ /**
+ * DN contains distinguished name
+ */
- Values?: string[],
+ DN?: string,
- ByteValues?: Uint8Array,
+ /**
+ * Attributes contains list of attributes
+ */
+
+ Attributes?: LdapAttributes,
}
@@ -584,12 +716,32 @@ export interface Metadata {
/**
- * SearchResult Interface
+ * SearchResult contains search result of any / all ldap search request
+ * @example
+ * ```javascript
+ * const ldap = require('nuclei/ldap');
+ * const client = new ldap.Client('ldap://ldap.example.com', 'acme.com');
+ * const results = client.Search('(objectinterface=*)', 'cn', 'mail');
+ * ```
*/
export interface SearchResult {
+ /**
+ * Referrals contains list of referrals
+ */
+
Referrals?: string[],
- Entries?: Entry,
+ /**
+ * Controls contains list of controls
+ */
+
+ Controls?: string[],
+
+ /**
+ * Entries contains list of entries
+ */
+
+ Entries?: LdapEntry[],
}
diff --git a/pkg/js/generated/ts/rdp.ts b/pkg/js/generated/ts/rdp.ts
index 77dfeafb13..7858b78319 100755
--- a/pkg/js/generated/ts/rdp.ts
+++ b/pkg/js/generated/ts/rdp.ts
@@ -78,8 +78,6 @@ export interface IsRDPResponse {
*/
export interface ServiceRDP {
- DNSDomainName?: string,
-
ForestName?: string,
OSFingerprint?: string,
@@ -93,5 +91,7 @@ export interface ServiceRDP {
NetBIOSDomainName?: string,
DNSComputerName?: string,
+
+ DNSDomainName?: string,
}
diff --git a/pkg/js/generated/ts/smb.ts b/pkg/js/generated/ts/smb.ts
index b0ca997ac5..c9b769cb5d 100755
--- a/pkg/js/generated/ts/smb.ts
+++ b/pkg/js/generated/ts/smb.ts
@@ -137,6 +137,10 @@ export interface NegotiationLog {
*/
export interface SMBCapabilities {
+ DFSSupport?: boolean,
+
+ Leasing?: boolean,
+
LargeMTU?: boolean,
MultiChan?: boolean,
@@ -146,10 +150,6 @@ export interface SMBCapabilities {
DirLeasing?: boolean,
Encryption?: boolean,
-
- DFSSupport?: boolean,
-
- Leasing?: boolean,
}
@@ -159,16 +159,16 @@ export interface SMBCapabilities {
*/
export interface SMBLog {
- SupportV1?: boolean,
-
- NativeOs?: string,
-
NTLM?: string,
GroupName?: string,
HasNTLM?: boolean,
+ SupportV1?: boolean,
+
+ NativeOs?: string,
+
Version?: SMBVersions,
Capabilities?: SMBCapabilities,
@@ -185,13 +185,13 @@ export interface SMBLog {
*/
export interface SMBVersions {
- VerString?: string,
-
Major?: number,
Minor?: number,
Revision?: number,
+
+ VerString?: string,
}
@@ -225,12 +225,12 @@ export interface ServiceSMB {
*/
export interface SessionSetupLog {
- NegotiateFlags?: number,
-
SetupFlags?: number,
TargetName?: string,
+ NegotiateFlags?: number,
+
HeaderLog?: HeaderLog,
}
diff --git a/pkg/js/generated/ts/ssh.ts b/pkg/js/generated/ts/ssh.ts
index 21dacf4054..058df03c5b 100755
--- a/pkg/js/generated/ts/ssh.ts
+++ b/pkg/js/generated/ts/ssh.ts
@@ -133,9 +133,9 @@ export interface Algorithms {
HostKey?: string,
- R?: DirectionAlgorithms,
-
W?: DirectionAlgorithms,
+
+ R?: DirectionAlgorithms,
}
@@ -159,13 +159,13 @@ export interface DirectionAlgorithms {
*/
export interface EndpointId {
- Raw?: string,
-
- ProtoVersion?: string,
-
SoftwareVersion?: string,
Comment?: string,
+
+ Raw?: string,
+
+ ProtoVersion?: string,
}
@@ -197,34 +197,34 @@ export interface HandshakeLog {
*/
export interface KexInitMsg {
- Reserved?: number,
-
- /**
- * fixed size array of length: [16]
- */
-
- Cookie?: Uint8Array,
+ KexAlgos?: string[],
CiphersClientServer?: string[],
- MACsClientServer?: string[],
-
MACsServerClient?: string[],
+ LanguagesClientServer?: string[],
+
+ CompressionClientServer?: string[],
+
CompressionServerClient?: string[],
- LanguagesClientServer?: string[],
+ Reserved?: number,
- FirstKexFollows?: boolean,
+ MACsClientServer?: string[],
- KexAlgos?: string[],
+ /**
+ * fixed size array of length: [16]
+ */
- CiphersServerClient?: string[],
+ Cookie?: Uint8Array,
- CompressionClientServer?: string[],
+ ServerHostKeyAlgos?: string[],
+
+ CiphersServerClient?: string[],
LanguagesServerClient?: string[],
- ServerHostKeyAlgos?: string[],
+ FirstKexFollows?: boolean,
}
diff --git a/pkg/js/libs/ldap/adenum.go b/pkg/js/libs/ldap/adenum.go
index 9aea98be97..849e4d6a15 100644
--- a/pkg/js/libs/ldap/adenum.go
+++ b/pkg/js/libs/ldap/adenum.go
@@ -74,25 +74,6 @@ func NegativeFilter(filter string) string {
return fmt.Sprintf("(!%s)", filter)
}
-type (
- // ADObject represents an Active Directory object
- // @example
- // ```javascript
- // const ldap = require('nuclei/ldap');
- // const client = new ldap.Client('ldap://ldap.example.com', 'acme.com');
- // const users = client.GetADUsers();
- // log(to_json(users));
- // ```
- ADObject struct {
- DistinguishedName string
- SAMAccountName string
- PWDLastSet string
- LastLogon string
- MemberOf []string
- ServicePrincipalName []string
- }
-)
-
// FindADObjects finds AD objects based on a filter
// and returns them as a list of ADObject
// @example
@@ -102,7 +83,7 @@ type (
// const users = client.FindADObjects(ldap.FilterIsPerson);
// log(to_json(users));
// ```
-func (c *Client) FindADObjects(filter string) []ADObject {
+func (c *Client) FindADObjects(filter string) SearchResult {
c.nj.Require(c.conn != nil, "no existing connection")
sr := ldap.NewSearchRequest(
c.BaseDN, ldap.ScopeWholeSubtree,
@@ -121,19 +102,7 @@ func (c *Client) FindADObjects(filter string) []ADObject {
res, err := c.conn.Search(sr)
c.nj.HandleError(err, "ldap search request failed")
-
- var objects []ADObject
- for _, obj := range res.Entries {
- objects = append(objects, ADObject{
- DistinguishedName: obj.GetAttributeValue("distinguishedName"),
- SAMAccountName: obj.GetAttributeValue("sAMAccountName"),
- PWDLastSet: DecodeADTimestamp(obj.GetAttributeValue("pwdLastSet")),
- LastLogon: DecodeADTimestamp(obj.GetAttributeValue("lastLogon")),
- MemberOf: obj.GetAttributeValues("memberOf"),
- ServicePrincipalName: obj.GetAttributeValues("servicePrincipalName"),
- })
- }
- return objects
+ return *getSearchResult(res)
}
// GetADUsers returns all AD users
@@ -145,7 +114,7 @@ func (c *Client) FindADObjects(filter string) []ADObject {
// const users = client.GetADUsers();
// log(to_json(users));
// ```
-func (c *Client) GetADUsers() []ADObject {
+func (c *Client) GetADUsers() SearchResult {
return c.FindADObjects(FilterIsPerson)
}
@@ -158,7 +127,7 @@ func (c *Client) GetADUsers() []ADObject {
// const users = client.GetADActiveUsers();
// log(to_json(users));
// ```
-func (c *Client) GetADActiveUsers() []ADObject {
+func (c *Client) GetADActiveUsers() SearchResult {
return c.FindADObjects(JoinFilters(FilterIsPerson, FilterAccountEnabled))
}
@@ -171,7 +140,7 @@ func (c *Client) GetADActiveUsers() []ADObject {
// const users = client.GetADUserWithNeverExpiringPasswords();
// log(to_json(users));
// ```
-func (c *Client) GetADUserWithNeverExpiringPasswords() []ADObject {
+func (c *Client) GetADUserWithNeverExpiringPasswords() SearchResult {
return c.FindADObjects(JoinFilters(FilterIsPerson, FilterDontExpirePassword))
}
@@ -184,7 +153,7 @@ func (c *Client) GetADUserWithNeverExpiringPasswords() []ADObject {
// const users = client.GetADUserTrustedForDelegation();
// log(to_json(users));
// ```
-func (c *Client) GetADUserTrustedForDelegation() []ADObject {
+func (c *Client) GetADUserTrustedForDelegation() SearchResult {
return c.FindADObjects(JoinFilters(FilterIsPerson, FilterTrustedForDelegation))
}
@@ -197,7 +166,7 @@ func (c *Client) GetADUserTrustedForDelegation() []ADObject {
// const users = client.GetADUserWithPasswordNotRequired();
// log(to_json(users));
// ```
-func (c *Client) GetADUserWithPasswordNotRequired() []ADObject {
+func (c *Client) GetADUserWithPasswordNotRequired() SearchResult {
return c.FindADObjects(JoinFilters(FilterIsPerson, FilterPasswordNotRequired))
}
@@ -210,7 +179,7 @@ func (c *Client) GetADUserWithPasswordNotRequired() []ADObject {
// const groups = client.GetADGroups();
// log(to_json(groups));
// ```
-func (c *Client) GetADGroups() []ADObject {
+func (c *Client) GetADGroups() SearchResult {
return c.FindADObjects(FilterIsGroup)
}
@@ -223,7 +192,7 @@ func (c *Client) GetADGroups() []ADObject {
// const dcs = client.GetADDCList();
// log(to_json(dcs));
// ```
-func (c *Client) GetADDCList() []ADObject {
+func (c *Client) GetADDCList() SearchResult {
return c.FindADObjects(JoinFilters(FilterIsComputer, FilterAccountEnabled, FilterServerTrustAccount))
}
@@ -236,7 +205,7 @@ func (c *Client) GetADDCList() []ADObject {
// const admins = client.GetADAdmins();
// log(to_json(admins));
// ```
-func (c *Client) GetADAdmins() []ADObject {
+func (c *Client) GetADAdmins() SearchResult {
return c.FindADObjects(JoinFilters(FilterIsPerson, FilterAccountEnabled, FilterIsAdmin))
}
@@ -249,7 +218,7 @@ func (c *Client) GetADAdmins() []ADObject {
// const kerberoastable = client.GetADUserKerberoastable();
// log(to_json(kerberoastable));
// ```
-func (c *Client) GetADUserKerberoastable() []ADObject {
+func (c *Client) GetADUserKerberoastable() SearchResult {
return c.FindADObjects(JoinFilters(FilterIsPerson, FilterAccountEnabled, FilterHasServicePrincipalName))
}
@@ -262,7 +231,7 @@ func (c *Client) GetADUserKerberoastable() []ADObject {
// const AsRepRoastable = client.GetADUserAsRepRoastable();
// log(to_json(AsRepRoastable));
// ```
-func (c *Client) GetADUserAsRepRoastable() []ADObject {
+func (c *Client) GetADUserAsRepRoastable() SearchResult {
return c.FindADObjects(JoinFilters(FilterIsPerson, FilterDontRequirePreauth))
}
@@ -276,7 +245,16 @@ func (c *Client) GetADUserAsRepRoastable() []ADObject {
// ```
func (c *Client) GetADDomainSID() string {
r := c.Search(FilterServerTrustAccount, "objectSid")
- c.nj.Require(len(r) > 0, "no result from GetADDomainSID query")
- c.nj.Require(len(r[0]["objectSid"]) > 0, "could not grab DomainSID")
- return DecodeSID(r[0]["objectSid"][0])
+ c.nj.Require(len(r.Entries) > 0, "no result from GetADDomainSID query")
+ for _, entry := range r.Entries {
+ if sid, ok := entry.Attributes.Extra["objectSid"]; ok {
+ if sid, ok := sid.([]string); ok {
+ return DecodeSID(sid[0])
+ } else {
+ c.nj.HandleError(fmt.Errorf("invalid objectSid type: %T", entry.Attributes.Extra["objectSid"]), "invalid objectSid type")
+ }
+ }
+ }
+ c.nj.HandleError(fmt.Errorf("no objectSid found"), "no objectSid found")
+ return ""
}
diff --git a/pkg/js/libs/ldap/ldap.go b/pkg/js/libs/ldap/ldap.go
index 31afa232d0..3961fa7b7e 100644
--- a/pkg/js/libs/ldap/ldap.go
+++ b/pkg/js/libs/ldap/ldap.go
@@ -203,7 +203,7 @@ func (c *Client) AuthenticateWithNTLMHash(username, hash string) {
// const client = new ldap.Client('ldap://ldap.example.com', 'acme.com');
// const results = client.Search('(objectClass=*)', 'cn', 'mail');
// ```
-func (c *Client) Search(filter string, attributes ...string) []map[string][]string {
+func (c *Client) Search(filter string, attributes ...string) SearchResult {
c.nj.Require(c.conn != nil, "no existing connection")
c.nj.Require(c.BaseDN != "", "base dn cannot be empty")
c.nj.Require(len(attributes) > 0, "attributes cannot be empty")
@@ -211,7 +211,7 @@ func (c *Client) Search(filter string, attributes ...string) []map[string][]stri
res, err := c.conn.Search(
ldap.NewSearchRequest(
"",
- ldap.ScopeBaseObject,
+ ldap.ScopeWholeSubtree,
ldap.NeverDerefAliases,
0, 0, false,
filter,
@@ -220,28 +220,7 @@ func (c *Client) Search(filter string, attributes ...string) []map[string][]stri
),
)
c.nj.HandleError(err, "ldap search request failed")
- if len(res.Entries) == 0 {
- // return empty list
- return nil
- }
-
- // convert ldap.Entry to []map[string][]string
- var out []map[string][]string
- for _, r := range res.Entries {
- app := make(map[string][]string)
- empty := true
- for _, a := range attributes {
- v := r.GetAttributeValues(a)
- if len(v) > 0 {
- app[a] = v
- empty = false
- }
- }
- if !empty {
- out = append(out, app)
- }
- }
- return out
+ return *getSearchResult(res)
}
// AdvancedSearch accepts all values of search request type and return Ldap Entry
@@ -257,7 +236,7 @@ func (c *Client) AdvancedSearch(
TypesOnly bool,
Filter string,
Attributes []string,
- Controls []ldap.Control) ldap.SearchResult {
+ Controls []ldap.Control) SearchResult {
c.nj.Require(c.conn != nil, "no existing connection")
if c.BaseDN == "" {
c.BaseDN = fmt.Sprintf("dc=%s", strings.Join(strings.Split(c.Realm, "."), ",dc="))
@@ -266,7 +245,7 @@ func (c *Client) AdvancedSearch(
res, err := c.conn.Search(req)
c.nj.HandleError(err, "ldap search request failed")
c.nj.Require(res != nil, "ldap search request failed got nil response")
- return *res
+ return *getSearchResult(res)
}
type (
@@ -302,7 +281,7 @@ func (c *Client) CollectMetadata() Metadata {
srMetadata := ldap.NewSearchRequest(
"",
- ldap.ScopeBaseObject,
+ ldap.ScopeWholeSubtree,
ldap.NeverDerefAliases,
0, 0, false,
"(objectClass=*)",
diff --git a/pkg/js/libs/ldap/utils.go b/pkg/js/libs/ldap/utils.go
index 96a243bbc7..bcff41dc49 100644
--- a/pkg/js/libs/ldap/utils.go
+++ b/pkg/js/libs/ldap/utils.go
@@ -5,8 +5,189 @@ import (
"strconv"
"strings"
"time"
+
+ "github.com/go-ldap/ldap/v3"
)
+type (
+ // SearchResult contains search result of any / all ldap search request
+ // @example
+ // ```javascript
+ // const ldap = require('nuclei/ldap');
+ // const client = new ldap.Client('ldap://ldap.example.com', 'acme.com');
+ // const results = client.Search('(objectClass=*)', 'cn', 'mail');
+ // ```
+ SearchResult struct {
+ // Referrals contains list of referrals
+ Referrals []string `json:"referrals"`
+ // Controls contains list of controls
+ Controls []string `json:"controls"`
+ // Entries contains list of entries
+ Entries []LdapEntry `json:"entries"`
+ }
+
+ // LdapEntry represents a single LDAP entry
+ LdapEntry struct {
+ // DN contains distinguished name
+ DN string `json:"dn"`
+ // Attributes contains list of attributes
+ Attributes LdapAttributes `json:"attributes"`
+ }
+
+ // LdapAttributes represents all LDAP attributes of a particular
+ // ldap entry
+ LdapAttributes struct {
+ // CurrentTime contains current time
+ CurrentTime []string `json:"currentTime,omitempty"`
+ // SubschemaSubentry contains subschema subentry
+ SubschemaSubentry []string `json:"subschemaSubentry,omitempty"`
+ // DsServiceName contains ds service name
+ DsServiceName []string `json:"dsServiceName,omitempty"`
+ // NamingContexts contains naming contexts
+ NamingContexts []string `json:"namingContexts,omitempty"`
+ // DefaultNamingContext contains default naming context
+ DefaultNamingContext []string `json:"defaultNamingContext,omitempty"`
+ // SchemaNamingContext contains schema naming context
+ SchemaNamingContext []string `json:"schemaNamingContext,omitempty"`
+ // ConfigurationNamingContext contains configuration naming context
+ ConfigurationNamingContext []string `json:"configurationNamingContext,omitempty"`
+ // RootDomainNamingContext contains root domain naming context
+ RootDomainNamingContext []string `json:"rootDomainNamingContext,omitempty"`
+ // SupportedLDAPVersion contains supported LDAP version
+ SupportedLDAPVersion []string `json:"supportedLDAPVersion,omitempty"`
+ // HighestCommittedUSN contains highest committed USN
+ HighestCommittedUSN []string `json:"highestCommittedUSN,omitempty"`
+ // SupportedSASLMechanisms contains supported SASL mechanisms
+ SupportedSASLMechanisms []string `json:"supportedSASLMechanisms,omitempty"`
+ // DnsHostName contains DNS host name
+ DnsHostName []string `json:"dnsHostName,omitempty"`
+ // LdapServiceName contains LDAP service name
+ LdapServiceName []string `json:"ldapServiceName,omitempty"`
+ // ServerName contains server name
+ ServerName []string `json:"serverName,omitempty"`
+ // IsSynchronized contains is synchronized
+ IsSynchronized []string `json:"isSynchronized,omitempty"`
+ // IsGlobalCatalogReady contains is global catalog ready
+ IsGlobalCatalogReady []string `json:"isGlobalCatalogReady,omitempty"`
+ // DomainFunctionality contains domain functionality
+ DomainFunctionality []string `json:"domainFunctionality,omitempty"`
+ // ForestFunctionality contains forest functionality
+ ForestFunctionality []string `json:"forestFunctionality,omitempty"`
+ // DomainControllerFunctionality contains domain controller functionality
+ DomainControllerFunctionality []string `json:"domainControllerFunctionality,omitempty"`
+ // DistinguishedName contains the distinguished name
+ DistinguishedName []string `json:"distinguishedName,omitempty"`
+ // SAMAccountName contains the SAM account name
+ SAMAccountName []string `json:"sAMAccountName,omitempty"`
+ // PWDLastSet contains the password last set time
+ PWDLastSet []string `json:"pwdLastSet,omitempty"`
+ // LastLogon contains the last logon time
+ LastLogon []string `json:"lastLogon,omitempty"`
+ // MemberOf contains the groups the entry is a member of
+ MemberOf []string `json:"memberOf,omitempty"`
+ // ServicePrincipalName contains the service principal names
+ ServicePrincipalName []string `json:"servicePrincipalName,omitempty"`
+ // Extra contains other extra fields which might be present
+ Extra map[string]any `json:"extra,omitempty"`
+ }
+)
+
+// getSearchResult converts a ldap.SearchResult to a SearchResult
+func getSearchResult(sr *ldap.SearchResult) *SearchResult {
+ t := &SearchResult{
+ Referrals: []string{},
+ Controls: []string{},
+ Entries: []LdapEntry{},
+ }
+ // add referrals
+ t.Referrals = append(t.Referrals, sr.Referrals...)
+ // add controls
+ for _, ctrl := range sr.Controls {
+ t.Controls = append(t.Controls, ctrl.String())
+ }
+ // add entries
+ for _, entry := range sr.Entries {
+ t.Entries = append(t.Entries, parseLdapEntry(entry))
+ }
+ return t
+}
+
+func parseLdapEntry(entry *ldap.Entry) LdapEntry {
+ e := LdapEntry{
+ DN: entry.DN,
+ }
+ attrs := LdapAttributes{
+ Extra: make(map[string]any),
+ }
+ for _, attr := range entry.Attributes {
+ switch attr.Name {
+ case "currentTime":
+ attrs.CurrentTime = decodeTimestamps(attr.Values)
+ case "subschemaSubentry":
+ attrs.SubschemaSubentry = attr.Values
+ case "dsServiceName":
+ attrs.DsServiceName = attr.Values
+ case "namingContexts":
+ attrs.NamingContexts = attr.Values
+ case "defaultNamingContext":
+ attrs.DefaultNamingContext = attr.Values
+ case "schemaNamingContext":
+ attrs.SchemaNamingContext = attr.Values
+ case "configurationNamingContext":
+ attrs.ConfigurationNamingContext = attr.Values
+ case "rootDomainNamingContext":
+ attrs.RootDomainNamingContext = attr.Values
+ case "supportedLDAPVersion":
+ attrs.SupportedLDAPVersion = attr.Values
+ case "highestCommittedUSN":
+ attrs.HighestCommittedUSN = attr.Values
+ case "supportedSASLMechanisms":
+ attrs.SupportedSASLMechanisms = attr.Values
+ case "dnsHostName":
+ attrs.DnsHostName = attr.Values
+ case "ldapServiceName":
+ attrs.LdapServiceName = attr.Values
+ case "serverName":
+ attrs.ServerName = attr.Values
+ case "isSynchronized":
+ attrs.IsSynchronized = attr.Values
+ case "isGlobalCatalogReady":
+ attrs.IsGlobalCatalogReady = attr.Values
+ case "domainFunctionality":
+ attrs.DomainFunctionality = attr.Values
+ case "forestFunctionality":
+ attrs.ForestFunctionality = attr.Values
+ case "domainControllerFunctionality":
+ attrs.DomainControllerFunctionality = attr.Values
+ case "distinguishedName":
+ attrs.DistinguishedName = attr.Values
+ case "sAMAccountName":
+ attrs.SAMAccountName = attr.Values
+ case "pwdLastSet":
+ attrs.PWDLastSet = decodeTimestamps(attr.Values)
+ case "lastLogon":
+ attrs.LastLogon = decodeTimestamps(attr.Values)
+ case "memberOf":
+ attrs.MemberOf = attr.Values
+ case "servicePrincipalName":
+ attrs.ServicePrincipalName = attr.Values
+ default:
+ attrs.Extra[attr.Name] = attr.Values
+ }
+ }
+ e.Attributes = attrs
+ return e
+}
+
+// decodeTimestamps decodes multiple timestamps
+func decodeTimestamps(timestamps []string) []string {
+ res := []string{}
+ for _, timestamp := range timestamps {
+ res = append(res, DecodeADTimestamp(timestamp))
+ }
+ return res
+}
+
// DecodeSID decodes a SID string
// @example
// ```javascript
From 6c32eb7d1e84155d7a59bbf503c6c261621d7ec9 Mon Sep 17 00:00:00 2001
From: Dogan Can Bakir <65292895+dogancanbakir@users.noreply.github.com>
Date: Mon, 15 Jul 2024 16:21:16 +0300
Subject: [PATCH 097/283] add sdk option to disable update check (#5346)
---
examples/simple/simple.go | 8 ++++++--
lib/config.go | 8 ++++++++
2 files changed, 14 insertions(+), 2 deletions(-)
diff --git a/examples/simple/simple.go b/examples/simple/simple.go
index 42d3fe4387..29de464655 100644
--- a/examples/simple/simple.go
+++ b/examples/simple/simple.go
@@ -1,9 +1,13 @@
package main
-import nuclei "github.com/projectdiscovery/nuclei/v3/lib"
+import (
+ "context"
+
+ nuclei "github.com/projectdiscovery/nuclei/v3/lib"
+)
func main() {
- ne, err := nuclei.NewNucleiEngine(
+ ne, err := nuclei.NewNucleiEngineCtx(context.Background(),
nuclei.WithTemplateFilters(nuclei.TemplateFilters{Tags: []string{"oast"}}),
nuclei.EnableStatsWithOpts(nuclei.StatsOptions{MetricServerPort: 6064}), // optionally enable metrics server for better observability
)
diff --git a/lib/config.go b/lib/config.go
index 24bc08691f..df44566225 100644
--- a/lib/config.go
+++ b/lib/config.go
@@ -440,3 +440,11 @@ func WithCatalog(cat catalog.Catalog) NucleiSDKOptions {
return nil
}
}
+
+// DisableUpdateCheck disables nuclei update check
+func DisableUpdateCheck() NucleiSDKOptions {
+ return func(e *NucleiEngine) error {
+ DefaultConfig.DisableUpdateCheck()
+ return nil
+ }
+}
From efc01c90c55d3147cae60a7b8e9b332436c0291a Mon Sep 17 00:00:00 2001
From: sandeep <8293321+ehsandeep@users.noreply.github.com>
Date: Mon, 15 Jul 2024 21:58:53 +0530
Subject: [PATCH 098/283] version update
---
pkg/catalog/config/constants.go | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/pkg/catalog/config/constants.go b/pkg/catalog/config/constants.go
index 113cb11509..8aa6beedfe 100644
--- a/pkg/catalog/config/constants.go
+++ b/pkg/catalog/config/constants.go
@@ -31,7 +31,7 @@ const (
CLIConfigFileName = "config.yaml"
ReportingConfigFilename = "reporting-config.yaml"
// Version is the current version of nuclei
- Version = `v3.3.0-dev`
+ Version = `v3.3.0`
// Directory Names of custom templates
CustomS3TemplatesDirName = "s3"
CustomGitHubTemplatesDirName = "github"
From f1253e16c91b1bc1e5ea971ca5693ff438d60088 Mon Sep 17 00:00:00 2001
From: sandeep <8293321+ehsandeep@users.noreply.github.com>
Date: Wed, 17 Jul 2024 14:03:03 +0530
Subject: [PATCH 099/283] readme updates
---
README.md | 11 +++++------
1 file changed, 5 insertions(+), 6 deletions(-)
diff --git a/README.md b/README.md
index 26b3d4a727..6f69782e05 100644
--- a/README.md
+++ b/README.md
@@ -203,7 +203,6 @@ CONFIGURATIONS:
-sml, -show-match-line show match lines for file templates, works with extractors only
-ztls use ztls library with autofallback to standard one for tls13 [Deprecated] autofallback to ztls is enabled by default
-sni string tls sni hostname to use (default: input domain name)
- -dt, -dialer-timeout value timeout for network requests.
-dka, -dialer-keep-alive value keep-alive duration for network requests.
-lfa, -allow-local-file-access allows file (payload) access anywhere on the system
-lna, -restrict-local-network-access blocks connections to the local / private network
@@ -212,7 +211,6 @@ CONFIGURATIONS:
-sip, -source-ip string source ip address to use for network scan
-rsr, -response-size-read int max response size to read in bytes
-rss, -response-size-save int max response size to read in bytes (default 1048576)
- -rrt, -response-read-timeout value response read timeout in seconds (default 5s)
-reset reset removes all nuclei configuration and data files (including nuclei-templates)
-tlsi, -tls-impersonate enable experimental client hello (ja3) tls randomization
-hae, -http-api-endpoint string experimental http api endpoint
@@ -238,7 +236,7 @@ FUZZING:
UNCOVER:
-uc, -uncover enable uncover engine
-uq, -uncover-query string[] uncover search query
- -ue, -uncover-engine string[] uncover search engine (shodan,censys,fofa,shodan-idb,quake,hunter,zoomeye,netlas,criminalip,publicwww,hunterhow) (default shodan)
+ -ue, -uncover-engine string[] uncover search engine (shodan,censys,fofa,shodan-idb,quake,hunter,zoomeye,netlas,criminalip,publicwww,hunterhow,google) (default shodan)
-uf, -uncover-field string uncover fields to return (ip,port,host) (default "ip:port")
-ul, -uncover-limit int uncover results to return (default 100)
-ur, -uncover-ratelimit int override ratelimit of engines with unknown ratelimit (default 60 req/min) (default 60)
@@ -311,9 +309,10 @@ STATISTICS:
-mp, -metrics-port int port to expose nuclei metrics on (default 9092)
CLOUD:
- -auth configure projectdiscovery cloud (pdcp) api key
- -cup, -cloud-upload upload scan results to pdcp dashboard
- -sid, -scan-id string upload scan results to given scan id
+ -auth configure projectdiscovery cloud (pdcp) api key (default true)
+ -cup, -cloud-upload upload scan results to pdcp dashboard
+ -sid, -scan-id string upload scan results to existing scan id (optional)
+ -sname, -scan-name string scan name to set (optional)
AUTHENTICATION:
-sf, -secret-file string[] path to config file containing secrets for nuclei authenticated scan
From 0433c83730eb50562bf6503b0520354d50ece6a9 Mon Sep 17 00:00:00 2001
From: sandeep <8293321+ehsandeep@users.noreply.github.com>
Date: Wed, 17 Jul 2024 18:36:05 +0530
Subject: [PATCH 100/283] dep update
---
go.mod | 6 +++---
go.sum | 12 ++++++------
2 files changed, 9 insertions(+), 9 deletions(-)
diff --git a/go.mod b/go.mod
index 67cfe79a80..d80fb0a03a 100644
--- a/go.mod
+++ b/go.mod
@@ -24,7 +24,7 @@ require (
github.com/projectdiscovery/hmap v0.0.49
github.com/projectdiscovery/interactsh v1.2.0
github.com/projectdiscovery/rawhttp v0.1.57
- github.com/projectdiscovery/retryabledns v1.0.65
+ github.com/projectdiscovery/retryabledns v1.0.66
github.com/projectdiscovery/retryablehttp-go v1.0.68
github.com/projectdiscovery/yamldoc-go v1.0.4
github.com/remeh/sizedwaitgroup v1.0.0
@@ -94,8 +94,8 @@ require (
github.com/projectdiscovery/sarif v0.0.1
github.com/projectdiscovery/tlsx v1.1.6
github.com/projectdiscovery/uncover v1.0.8
- github.com/projectdiscovery/useragent v0.0.57
- github.com/projectdiscovery/utils v0.2.0
+ github.com/projectdiscovery/useragent v0.0.60
+ github.com/projectdiscovery/utils v0.2.1
github.com/projectdiscovery/wappalyzergo v0.1.6
github.com/redis/go-redis/v9 v9.1.0
github.com/seh-msft/burpxml v1.0.1
diff --git a/go.sum b/go.sum
index c7c8ea430b..49bb48facf 100644
--- a/go.sum
+++ b/go.sum
@@ -874,8 +874,8 @@ github.com/projectdiscovery/rawhttp v0.1.57 h1:2vCT2i1NSZbTBH+uUBrxOJjxDPKgIl2q6
github.com/projectdiscovery/rawhttp v0.1.57/go.mod h1:qtthyaU0k8eqcEdza1R/fTqwyxSK4BZ511ThxgkiQtE=
github.com/projectdiscovery/rdap v0.9.1-0.20221108103045-9865884d1917 h1:m03X4gBVSorSzvmm0bFa7gDV4QNSOWPL/fgZ4kTXBxk=
github.com/projectdiscovery/rdap v0.9.1-0.20221108103045-9865884d1917/go.mod h1:JxXtZC9e195awe7EynrcnBJmFoad/BNDzW9mzFkK8Sg=
-github.com/projectdiscovery/retryabledns v1.0.65 h1:wOtMuVkrcrGkzaT8Jixb3BtUcJODXLRz+z9EjftmFsQ=
-github.com/projectdiscovery/retryabledns v1.0.65/go.mod h1:lf3xRdFhGh55fWVlzK1oAaDPrsKliqH3gGvsFNAOkCk=
+github.com/projectdiscovery/retryabledns v1.0.66 h1:OzKcR+TqIYdSDEED2JWJA90ZNFCEKRv+hOvpMHEZEOY=
+github.com/projectdiscovery/retryabledns v1.0.66/go.mod h1:ODpvZ6kXXB2uqK7GPyfkd/tR+4wkeIm1DR5TkRQv7KE=
github.com/projectdiscovery/retryablehttp-go v1.0.68 h1:MN/kfccDnebmp1kNcCnGlGuCfvet6y5eN1Y1hIKBB1I=
github.com/projectdiscovery/retryablehttp-go v1.0.68/go.mod h1:F3QNCoJExPFT59AACrVgYVhNiDW5Fp8s5HbXc+8CaaU=
github.com/projectdiscovery/sarif v0.0.1 h1:C2Tyj0SGOKbCLgHrx83vaE6YkzXEVrMXYRGLkKCr/us=
@@ -886,10 +886,10 @@ github.com/projectdiscovery/tlsx v1.1.6 h1:iw2zwKbd2+kRQ8J1G4dLmS0CLyemd/tKz1Uzc
github.com/projectdiscovery/tlsx v1.1.6/go.mod h1:s7SRRFdrwIZBK/RXXZi4CR/CubqFSvp8h5Bk1srEZIo=
github.com/projectdiscovery/uncover v1.0.8 h1:nE8bYJuwhqk0BEMRQRhNbjPGMy40A9gkiMk5xq6U4u0=
github.com/projectdiscovery/uncover v1.0.8/go.mod h1:1SwsNqjwMkJEzJQ7lQr5AHIdYd+BQlyqhO5IwIYmIAM=
-github.com/projectdiscovery/useragent v0.0.57 h1:y6ML3GVJ6Sq+nhqk2gY9aD0JNaKzfpunJPjPN40eVN4=
-github.com/projectdiscovery/useragent v0.0.57/go.mod h1:iChqL4rGApUsdXhAzItirEpMvYYOlruieZE+nqoKpCc=
-github.com/projectdiscovery/utils v0.2.0 h1:GC9FOq2mVacVYZgD2s6XW7RP6jAC/XjOKclSpLYPUUQ=
-github.com/projectdiscovery/utils v0.2.0/go.mod h1:kOnMoFfuQMY8U+nh8e32odjC00iEbiLVBcDxyAZFwCc=
+github.com/projectdiscovery/useragent v0.0.60 h1:qDU1rwA+XOKmSqp7yoijAN4PuvLQc2ZvuaoWH7QIkuc=
+github.com/projectdiscovery/useragent v0.0.60/go.mod h1:05IDiJEy2dWl3x6dnsWtJYPwT40oWha144Us7+Fwr6w=
+github.com/projectdiscovery/utils v0.2.1 h1:XcOUJtR5ReXDWqxDkJWPVpC2lSqIYTs8z8YNILcPiqk=
+github.com/projectdiscovery/utils v0.2.1/go.mod h1:h3o2zmJguUm9FKvel1F9jYCLxoWhFakyytRMjqb7Dg4=
github.com/projectdiscovery/wappalyzergo v0.1.6 h1:4iZDEnytsnAqjQgD5K6lmAzVx++KSQeMCDfomDtQM1M=
github.com/projectdiscovery/wappalyzergo v0.1.6/go.mod h1:wBYGKmA5BQp/NWsAy1q/jSH8N1LHWQ/LV26DuR+KzPM=
github.com/projectdiscovery/yamldoc-go v1.0.4 h1:eZoESapnMw6WAHiVgRwNqvbJEfNHEH148uthhFbG5jE=
From d776dbb2dd29f022e89fe0192a37d796eacadfe2 Mon Sep 17 00:00:00 2001
From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com>
Date: Wed, 17 Jul 2024 13:07:13 +0000
Subject: [PATCH 101/283] chore(deps): bump
github.com/projectdiscovery/wappalyzergo
Bumps [github.com/projectdiscovery/wappalyzergo](https://github.com/projectdiscovery/wappalyzergo) from 0.1.6 to 0.1.10.
- [Release notes](https://github.com/projectdiscovery/wappalyzergo/releases)
- [Commits](https://github.com/projectdiscovery/wappalyzergo/compare/v0.1.6...v0.1.10)
---
updated-dependencies:
- dependency-name: github.com/projectdiscovery/wappalyzergo
dependency-type: direct:production
update-type: version-update:semver-patch
...
Signed-off-by: dependabot[bot]
---
go.mod | 2 +-
go.sum | 4 ++--
2 files changed, 3 insertions(+), 3 deletions(-)
diff --git a/go.mod b/go.mod
index d80fb0a03a..e9efbb8f70 100644
--- a/go.mod
+++ b/go.mod
@@ -96,7 +96,7 @@ require (
github.com/projectdiscovery/uncover v1.0.8
github.com/projectdiscovery/useragent v0.0.60
github.com/projectdiscovery/utils v0.2.1
- github.com/projectdiscovery/wappalyzergo v0.1.6
+ github.com/projectdiscovery/wappalyzergo v0.1.10
github.com/redis/go-redis/v9 v9.1.0
github.com/seh-msft/burpxml v1.0.1
github.com/stretchr/testify v1.9.0
diff --git a/go.sum b/go.sum
index 49bb48facf..52dfe6a693 100644
--- a/go.sum
+++ b/go.sum
@@ -890,8 +890,8 @@ github.com/projectdiscovery/useragent v0.0.60 h1:qDU1rwA+XOKmSqp7yoijAN4PuvLQc2Z
github.com/projectdiscovery/useragent v0.0.60/go.mod h1:05IDiJEy2dWl3x6dnsWtJYPwT40oWha144Us7+Fwr6w=
github.com/projectdiscovery/utils v0.2.1 h1:XcOUJtR5ReXDWqxDkJWPVpC2lSqIYTs8z8YNILcPiqk=
github.com/projectdiscovery/utils v0.2.1/go.mod h1:h3o2zmJguUm9FKvel1F9jYCLxoWhFakyytRMjqb7Dg4=
-github.com/projectdiscovery/wappalyzergo v0.1.6 h1:4iZDEnytsnAqjQgD5K6lmAzVx++KSQeMCDfomDtQM1M=
-github.com/projectdiscovery/wappalyzergo v0.1.6/go.mod h1:wBYGKmA5BQp/NWsAy1q/jSH8N1LHWQ/LV26DuR+KzPM=
+github.com/projectdiscovery/wappalyzergo v0.1.10 h1:cUTMw8xYXyKTAxUiGYltJxEh4OtJyCvaahj+RG3mHSY=
+github.com/projectdiscovery/wappalyzergo v0.1.10/go.mod h1:/hzgxkBFTMe2wDbA93nFfoMjULw7/vIZ9QPSAnCgUa8=
github.com/projectdiscovery/yamldoc-go v1.0.4 h1:eZoESapnMw6WAHiVgRwNqvbJEfNHEH148uthhFbG5jE=
github.com/projectdiscovery/yamldoc-go v1.0.4/go.mod h1:8PIPRcUD55UbtQdcfFR1hpIGRWG0P7alClXNGt1TBik=
github.com/prometheus/client_golang v0.9.1/go.mod h1:7SWBe2y4D6OKWSNQJUaRYU/AaXPKyh/dDVn+NZz0KFw=
From 90bbecb51c798812494b0e86a0914edb79e2a61d Mon Sep 17 00:00:00 2001
From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com>
Date: Mon, 22 Jul 2024 05:59:47 +0000
Subject: [PATCH 102/283] chore(deps): bump github.com/projectdiscovery/httpx
from 1.6.3 to 1.6.6
Bumps [github.com/projectdiscovery/httpx](https://github.com/projectdiscovery/httpx) from 1.6.3 to 1.6.6.
- [Release notes](https://github.com/projectdiscovery/httpx/releases)
- [Changelog](https://github.com/projectdiscovery/httpx/blob/main/.goreleaser.yml)
- [Commits](https://github.com/projectdiscovery/httpx/compare/v1.6.3...v1.6.6)
---
updated-dependencies:
- dependency-name: github.com/projectdiscovery/httpx
dependency-type: direct:production
update-type: version-update:semver-patch
...
Signed-off-by: dependabot[bot]
---
go.mod | 2 +-
go.sum | 4 ++--
2 files changed, 3 insertions(+), 3 deletions(-)
diff --git a/go.mod b/go.mod
index e9efbb8f70..efb3dcbc5c 100644
--- a/go.mod
+++ b/go.mod
@@ -86,7 +86,7 @@ require (
github.com/projectdiscovery/gologger v1.1.14
github.com/projectdiscovery/gostruct v0.0.2
github.com/projectdiscovery/gozero v0.0.2
- github.com/projectdiscovery/httpx v1.6.3
+ github.com/projectdiscovery/httpx v1.6.6
github.com/projectdiscovery/mapcidr v1.1.34
github.com/projectdiscovery/n3iwf v0.0.0-20230523120440-b8cd232ff1f5
github.com/projectdiscovery/ratelimit v0.0.45
diff --git a/go.sum b/go.sum
index 52dfe6a693..ffa9cff5e4 100644
--- a/go.sum
+++ b/go.sum
@@ -854,8 +854,8 @@ github.com/projectdiscovery/gozero v0.0.2 h1:8fJeaCjxL9tpm33uG/RsCQs6HGM/NE6eA3c
github.com/projectdiscovery/gozero v0.0.2/go.mod h1:d8bZvDWW07LWNYWrwjZ4OO1I0cpkfqaysyDfSs9ibK8=
github.com/projectdiscovery/hmap v0.0.49 h1:QNW98JDqd0nmaIFRMbn0sLocRIGXAud1liGip4iPdvc=
github.com/projectdiscovery/hmap v0.0.49/go.mod h1:BA4cyYTrWKMUa78815R6p3tuGiYu2MJHYnjwXmefdbU=
-github.com/projectdiscovery/httpx v1.6.3 h1:TSu3zEKOHGFfMsGeB0lZEFRl6Tq4nPuIgKDXi9NnYpg=
-github.com/projectdiscovery/httpx v1.6.3/go.mod h1:9FmsmkT71aTa4WnOyBIPzB8I9c0QAJ9PSVUqs2NkHRQ=
+github.com/projectdiscovery/httpx v1.6.6 h1:e9deBDrW2ILhuHjPYBAskIgEaLTJYbiBjC5FAHKuV4A=
+github.com/projectdiscovery/httpx v1.6.6/go.mod h1:7kLxlw3gW6IJYwRbThI3rFkaQxJ/Z4zNeJtk408RnW0=
github.com/projectdiscovery/interactsh v1.2.0 h1:Al6jHiR+Usl9egYJDLJaWNHOcH8Rugk8gWMasc8Cmw8=
github.com/projectdiscovery/interactsh v1.2.0/go.mod h1:Wxt0fnzxsfrAZQQlpVrf3xMatP4OXZaZbjuDkIQKdYY=
github.com/projectdiscovery/ldapserver v1.0.2-0.20240219154113-dcc758ebc0cb h1:MGtI4oE12ruWv11ZlPXXd7hl/uAaQZrFvrIDYDeVMd8=
From d7d5adff5310a31c3dc0e7e7127b51d16ab77c51 Mon Sep 17 00:00:00 2001
From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com>
Date: Mon, 22 Jul 2024 05:59:52 +0000
Subject: [PATCH 103/283] chore(deps): bump
github.com/projectdiscovery/retryabledns
Bumps [github.com/projectdiscovery/retryabledns](https://github.com/projectdiscovery/retryabledns) from 1.0.66 to 1.0.68.
- [Release notes](https://github.com/projectdiscovery/retryabledns/releases)
- [Commits](https://github.com/projectdiscovery/retryabledns/compare/v1.0.66...v1.0.68)
---
updated-dependencies:
- dependency-name: github.com/projectdiscovery/retryabledns
dependency-type: direct:production
update-type: version-update:semver-patch
...
Signed-off-by: dependabot[bot]
---
go.mod | 2 +-
go.sum | 4 ++--
2 files changed, 3 insertions(+), 3 deletions(-)
diff --git a/go.mod b/go.mod
index e9efbb8f70..2b56c4faae 100644
--- a/go.mod
+++ b/go.mod
@@ -24,7 +24,7 @@ require (
github.com/projectdiscovery/hmap v0.0.49
github.com/projectdiscovery/interactsh v1.2.0
github.com/projectdiscovery/rawhttp v0.1.57
- github.com/projectdiscovery/retryabledns v1.0.66
+ github.com/projectdiscovery/retryabledns v1.0.68
github.com/projectdiscovery/retryablehttp-go v1.0.68
github.com/projectdiscovery/yamldoc-go v1.0.4
github.com/remeh/sizedwaitgroup v1.0.0
diff --git a/go.sum b/go.sum
index 52dfe6a693..17504cf76a 100644
--- a/go.sum
+++ b/go.sum
@@ -874,8 +874,8 @@ github.com/projectdiscovery/rawhttp v0.1.57 h1:2vCT2i1NSZbTBH+uUBrxOJjxDPKgIl2q6
github.com/projectdiscovery/rawhttp v0.1.57/go.mod h1:qtthyaU0k8eqcEdza1R/fTqwyxSK4BZ511ThxgkiQtE=
github.com/projectdiscovery/rdap v0.9.1-0.20221108103045-9865884d1917 h1:m03X4gBVSorSzvmm0bFa7gDV4QNSOWPL/fgZ4kTXBxk=
github.com/projectdiscovery/rdap v0.9.1-0.20221108103045-9865884d1917/go.mod h1:JxXtZC9e195awe7EynrcnBJmFoad/BNDzW9mzFkK8Sg=
-github.com/projectdiscovery/retryabledns v1.0.66 h1:OzKcR+TqIYdSDEED2JWJA90ZNFCEKRv+hOvpMHEZEOY=
-github.com/projectdiscovery/retryabledns v1.0.66/go.mod h1:ODpvZ6kXXB2uqK7GPyfkd/tR+4wkeIm1DR5TkRQv7KE=
+github.com/projectdiscovery/retryabledns v1.0.68 h1:EWWG7WsGTT0YvwIjHclIWSWgv4R29xMWShR6Yt5Z+pA=
+github.com/projectdiscovery/retryabledns v1.0.68/go.mod h1:72W9RwsHVRIGmtc4W6i6izVtYzKBTdnCE1VciqYM5Eg=
github.com/projectdiscovery/retryablehttp-go v1.0.68 h1:MN/kfccDnebmp1kNcCnGlGuCfvet6y5eN1Y1hIKBB1I=
github.com/projectdiscovery/retryablehttp-go v1.0.68/go.mod h1:F3QNCoJExPFT59AACrVgYVhNiDW5Fp8s5HbXc+8CaaU=
github.com/projectdiscovery/sarif v0.0.1 h1:C2Tyj0SGOKbCLgHrx83vaE6YkzXEVrMXYRGLkKCr/us=
From 327a15f94fe4fd185631437ff87291073ba49c3f Mon Sep 17 00:00:00 2001
From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com>
Date: Mon, 22 Jul 2024 06:27:14 +0000
Subject: [PATCH 104/283] chore(deps): bump github.com/projectdiscovery/hmap
from 0.0.49 to 0.0.51
Bumps [github.com/projectdiscovery/hmap](https://github.com/projectdiscovery/hmap) from 0.0.49 to 0.0.51.
- [Release notes](https://github.com/projectdiscovery/hmap/releases)
- [Commits](https://github.com/projectdiscovery/hmap/compare/v0.0.49...v0.0.51)
---
updated-dependencies:
- dependency-name: github.com/projectdiscovery/hmap
dependency-type: direct:production
update-type: version-update:semver-patch
...
Signed-off-by: dependabot[bot]
---
go.mod | 2 +-
go.sum | 4 ++--
2 files changed, 3 insertions(+), 3 deletions(-)
diff --git a/go.mod b/go.mod
index 9deb77f2b2..ad51ffbe5c 100644
--- a/go.mod
+++ b/go.mod
@@ -21,7 +21,7 @@ require (
github.com/pkg/errors v0.9.1
github.com/projectdiscovery/clistats v0.0.20
github.com/projectdiscovery/fastdialer v0.1.6
- github.com/projectdiscovery/hmap v0.0.49
+ github.com/projectdiscovery/hmap v0.0.51
github.com/projectdiscovery/interactsh v1.2.0
github.com/projectdiscovery/rawhttp v0.1.57
github.com/projectdiscovery/retryabledns v1.0.68
diff --git a/go.sum b/go.sum
index cd59ce47f6..b35498f019 100644
--- a/go.sum
+++ b/go.sum
@@ -852,8 +852,8 @@ github.com/projectdiscovery/gostruct v0.0.2 h1:s8gP8ApugGM4go1pA+sVlPDXaWqNP5BBD
github.com/projectdiscovery/gostruct v0.0.2/go.mod h1:H86peL4HKwMXcQQtEa6lmC8FuD9XFt6gkNR0B/Mu5PE=
github.com/projectdiscovery/gozero v0.0.2 h1:8fJeaCjxL9tpm33uG/RsCQs6HGM/NE6eA3cjkilRQ+E=
github.com/projectdiscovery/gozero v0.0.2/go.mod h1:d8bZvDWW07LWNYWrwjZ4OO1I0cpkfqaysyDfSs9ibK8=
-github.com/projectdiscovery/hmap v0.0.49 h1:QNW98JDqd0nmaIFRMbn0sLocRIGXAud1liGip4iPdvc=
-github.com/projectdiscovery/hmap v0.0.49/go.mod h1:BA4cyYTrWKMUa78815R6p3tuGiYu2MJHYnjwXmefdbU=
+github.com/projectdiscovery/hmap v0.0.51 h1:xqbpRAJRHPMoS2uERkbWGObIO4bv+whe3PEk3h4lDEg=
+github.com/projectdiscovery/hmap v0.0.51/go.mod h1:vqdeWnNVMJYyIDytu+IdJDFg3wZdRVN83AKHR40RP6c=
github.com/projectdiscovery/httpx v1.6.6 h1:e9deBDrW2ILhuHjPYBAskIgEaLTJYbiBjC5FAHKuV4A=
github.com/projectdiscovery/httpx v1.6.6/go.mod h1:7kLxlw3gW6IJYwRbThI3rFkaQxJ/Z4zNeJtk408RnW0=
github.com/projectdiscovery/interactsh v1.2.0 h1:Al6jHiR+Usl9egYJDLJaWNHOcH8Rugk8gWMasc8Cmw8=
From ec682695b6d55c3894cfc1bb25f68d8e56549f28 Mon Sep 17 00:00:00 2001
From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com>
Date: Mon, 22 Jul 2024 06:52:54 +0000
Subject: [PATCH 105/283] chore(deps): bump github.com/projectdiscovery/utils
from 0.2.1 to 0.2.2
Bumps [github.com/projectdiscovery/utils](https://github.com/projectdiscovery/utils) from 0.2.1 to 0.2.2.
- [Release notes](https://github.com/projectdiscovery/utils/releases)
- [Changelog](https://github.com/projectdiscovery/utils/blob/main/CHANGELOG.md)
- [Commits](https://github.com/projectdiscovery/utils/compare/v0.2.1...v0.2.2)
---
updated-dependencies:
- dependency-name: github.com/projectdiscovery/utils
dependency-type: direct:production
update-type: version-update:semver-patch
...
Signed-off-by: dependabot[bot]
---
go.mod | 10 +++++-----
go.sum | 20 ++++++++++----------
2 files changed, 15 insertions(+), 15 deletions(-)
diff --git a/go.mod b/go.mod
index ad51ffbe5c..c03677c803 100644
--- a/go.mod
+++ b/go.mod
@@ -20,12 +20,12 @@ require (
github.com/olekukonko/tablewriter v0.0.5
github.com/pkg/errors v0.9.1
github.com/projectdiscovery/clistats v0.0.20
- github.com/projectdiscovery/fastdialer v0.1.6
+ github.com/projectdiscovery/fastdialer v0.2.0
github.com/projectdiscovery/hmap v0.0.51
github.com/projectdiscovery/interactsh v1.2.0
github.com/projectdiscovery/rawhttp v0.1.57
github.com/projectdiscovery/retryabledns v1.0.68
- github.com/projectdiscovery/retryablehttp-go v1.0.68
+ github.com/projectdiscovery/retryablehttp-go v1.0.69
github.com/projectdiscovery/yamldoc-go v1.0.4
github.com/remeh/sizedwaitgroup v1.0.0
github.com/rs/xid v1.5.0
@@ -83,7 +83,7 @@ require (
github.com/projectdiscovery/fasttemplate v0.0.2
github.com/projectdiscovery/go-smb2 v0.0.0-20240129202741-052cc450c6cb
github.com/projectdiscovery/goflags v0.1.59
- github.com/projectdiscovery/gologger v1.1.14
+ github.com/projectdiscovery/gologger v1.1.15
github.com/projectdiscovery/gostruct v0.0.2
github.com/projectdiscovery/gozero v0.0.2
github.com/projectdiscovery/httpx v1.6.6
@@ -95,7 +95,7 @@ require (
github.com/projectdiscovery/tlsx v1.1.6
github.com/projectdiscovery/uncover v1.0.8
github.com/projectdiscovery/useragent v0.0.60
- github.com/projectdiscovery/utils v0.2.1
+ github.com/projectdiscovery/utils v0.2.2
github.com/projectdiscovery/wappalyzergo v0.1.10
github.com/redis/go-redis/v9 v9.1.0
github.com/seh-msft/burpxml v1.0.1
@@ -205,7 +205,7 @@ require (
github.com/projectdiscovery/freeport v0.0.5 // indirect
github.com/projectdiscovery/ldapserver v1.0.2-0.20240219154113-dcc758ebc0cb // indirect
github.com/projectdiscovery/machineid v0.0.0-20240226150047-2e2c51e35983 // indirect
- github.com/refraction-networking/utls v1.6.6 // indirect
+ github.com/refraction-networking/utls v1.6.7 // indirect
github.com/sashabaranov/go-openai v1.15.3 // indirect
github.com/shirou/gopsutil v3.21.11+incompatible // indirect
github.com/shoenig/go-m1cpu v0.1.6 // indirect
diff --git a/go.sum b/go.sum
index b35498f019..2172188616 100644
--- a/go.sum
+++ b/go.sum
@@ -836,8 +836,8 @@ github.com/projectdiscovery/clistats v0.0.20 h1:5jO5SLiRJ7f0nDV0ndBNmBeesbROouPo
github.com/projectdiscovery/clistats v0.0.20/go.mod h1:GJ2av0KnOvK0AISQnP8hyDclYIji1LVkx2l0pwnzAu4=
github.com/projectdiscovery/dsl v0.1.6 h1:6TUYMwbjcYJ5OoRoawPX1SYD7wgubX/1FreC8SwFqgc=
github.com/projectdiscovery/dsl v0.1.6/go.mod h1:1PAV9A6X+fdyMXW0SY4i0BHrO8yN2+FB2jp25+0AT04=
-github.com/projectdiscovery/fastdialer v0.1.6 h1:dtdE7WjaydpndKPcUYWms5iOhBSC0nvBAHyZd6I4pdQ=
-github.com/projectdiscovery/fastdialer v0.1.6/go.mod h1:a5XOjvaO4b+ddO/YLoM4nvK3CPPviW6HmoAd+AoszMw=
+github.com/projectdiscovery/fastdialer v0.2.0 h1:ZEYKA9L5VerrD9LcXH/gtTqUHiBQvTudrbqkZSRMpJo=
+github.com/projectdiscovery/fastdialer v0.2.0/go.mod h1:bwBv51HzxK7DkCjB1EHQJKDzXKGPDLp6MgK8PPjaW/w=
github.com/projectdiscovery/fasttemplate v0.0.2 h1:h2cISk5xDhlJEinlBQS6RRx0vOlOirB2y3Yu4PJzpiA=
github.com/projectdiscovery/fasttemplate v0.0.2/go.mod h1:XYWWVMxnItd+r0GbjA1GCsUopMw1/XusuQxdyAIHMCw=
github.com/projectdiscovery/freeport v0.0.5 h1:jnd3Oqsl4S8n0KuFkE5Hm8WGDP24ITBvmyw5pFTHS8Q=
@@ -846,8 +846,8 @@ github.com/projectdiscovery/go-smb2 v0.0.0-20240129202741-052cc450c6cb h1:rutG90
github.com/projectdiscovery/go-smb2 v0.0.0-20240129202741-052cc450c6cb/go.mod h1:FLjF1DmZ+POoGEiIQdWuYVwS++C/GwpX8YaCsTSm1RY=
github.com/projectdiscovery/goflags v0.1.59 h1:rFScpC57L0Ln0dJmBtstyxHUH7K9H4UmHACXcSkdqqk=
github.com/projectdiscovery/goflags v0.1.59/go.mod h1:dj67QGp/D81WRLtzU0HsWR20zgoGZ0cnk3Wbt9xJcuo=
-github.com/projectdiscovery/gologger v1.1.14 h1:8SS9qDCisCFffKHzWMX+GDDArxBJ9A7DhYrESEixAYo=
-github.com/projectdiscovery/gologger v1.1.14/go.mod h1:CPk1nAZ3PqgspKBuSJR+xwY2i/Rm+P/ovahpP30EJy8=
+github.com/projectdiscovery/gologger v1.1.15 h1:EgXC9uQkYb5qWcazV6mH0mev23MZ0GLiGyl203nDYbU=
+github.com/projectdiscovery/gologger v1.1.15/go.mod h1:qPDCXZIxqlaKYhX0iyXZtsbtTFU152knbNRF4Z6fXUQ=
github.com/projectdiscovery/gostruct v0.0.2 h1:s8gP8ApugGM4go1pA+sVlPDXaWqNP5BBDDSv7VEdG1M=
github.com/projectdiscovery/gostruct v0.0.2/go.mod h1:H86peL4HKwMXcQQtEa6lmC8FuD9XFt6gkNR0B/Mu5PE=
github.com/projectdiscovery/gozero v0.0.2 h1:8fJeaCjxL9tpm33uG/RsCQs6HGM/NE6eA3cjkilRQ+E=
@@ -876,8 +876,8 @@ github.com/projectdiscovery/rdap v0.9.1-0.20221108103045-9865884d1917 h1:m03X4gB
github.com/projectdiscovery/rdap v0.9.1-0.20221108103045-9865884d1917/go.mod h1:JxXtZC9e195awe7EynrcnBJmFoad/BNDzW9mzFkK8Sg=
github.com/projectdiscovery/retryabledns v1.0.68 h1:EWWG7WsGTT0YvwIjHclIWSWgv4R29xMWShR6Yt5Z+pA=
github.com/projectdiscovery/retryabledns v1.0.68/go.mod h1:72W9RwsHVRIGmtc4W6i6izVtYzKBTdnCE1VciqYM5Eg=
-github.com/projectdiscovery/retryablehttp-go v1.0.68 h1:MN/kfccDnebmp1kNcCnGlGuCfvet6y5eN1Y1hIKBB1I=
-github.com/projectdiscovery/retryablehttp-go v1.0.68/go.mod h1:F3QNCoJExPFT59AACrVgYVhNiDW5Fp8s5HbXc+8CaaU=
+github.com/projectdiscovery/retryablehttp-go v1.0.69 h1:EbbHtZS1g/aT7sM3ZFNMcvnMfe9wuqXX2rEG1vGFKe4=
+github.com/projectdiscovery/retryablehttp-go v1.0.69/go.mod h1:STJ0rpHJHofbAq22yI8nZCKTVWOk50xZ+oMJHL1rc5g=
github.com/projectdiscovery/sarif v0.0.1 h1:C2Tyj0SGOKbCLgHrx83vaE6YkzXEVrMXYRGLkKCr/us=
github.com/projectdiscovery/sarif v0.0.1/go.mod h1:cEYlDu8amcPf6b9dSakcz2nNnJsoz4aR6peERwV+wuQ=
github.com/projectdiscovery/stringsutil v0.0.2 h1:uzmw3IVLJSMW1kEg8eCStG/cGbYYZAja8BH3LqqJXMA=
@@ -888,8 +888,8 @@ github.com/projectdiscovery/uncover v1.0.8 h1:nE8bYJuwhqk0BEMRQRhNbjPGMy40A9gkiM
github.com/projectdiscovery/uncover v1.0.8/go.mod h1:1SwsNqjwMkJEzJQ7lQr5AHIdYd+BQlyqhO5IwIYmIAM=
github.com/projectdiscovery/useragent v0.0.60 h1:qDU1rwA+XOKmSqp7yoijAN4PuvLQc2ZvuaoWH7QIkuc=
github.com/projectdiscovery/useragent v0.0.60/go.mod h1:05IDiJEy2dWl3x6dnsWtJYPwT40oWha144Us7+Fwr6w=
-github.com/projectdiscovery/utils v0.2.1 h1:XcOUJtR5ReXDWqxDkJWPVpC2lSqIYTs8z8YNILcPiqk=
-github.com/projectdiscovery/utils v0.2.1/go.mod h1:h3o2zmJguUm9FKvel1F9jYCLxoWhFakyytRMjqb7Dg4=
+github.com/projectdiscovery/utils v0.2.2 h1:ZzanMTiVboM7vFO6cjt+GRDtvVjZD1GChBa7knekBgs=
+github.com/projectdiscovery/utils v0.2.2/go.mod h1:sZHBCrS37ejIb0/pFjKZewm1nBWMTXUsLJH3hTVww0Y=
github.com/projectdiscovery/wappalyzergo v0.1.10 h1:cUTMw8xYXyKTAxUiGYltJxEh4OtJyCvaahj+RG3mHSY=
github.com/projectdiscovery/wappalyzergo v0.1.10/go.mod h1:/hzgxkBFTMe2wDbA93nFfoMjULw7/vIZ9QPSAnCgUa8=
github.com/projectdiscovery/yamldoc-go v1.0.4 h1:eZoESapnMw6WAHiVgRwNqvbJEfNHEH148uthhFbG5jE=
@@ -929,8 +929,8 @@ github.com/prometheus/procfs v0.9.0/go.mod h1:+pB4zwohETzFnmlpe6yd2lSc+0/46IYZRB
github.com/rcrowley/go-metrics v0.0.0-20181016184325-3113b8401b8a/go.mod h1:bCqnVzQkZxMG4s8nGwiZ5l3QUCyqpo9Y+/ZMZ9VjZe4=
github.com/redis/go-redis/v9 v9.1.0 h1:137FnGdk+EQdCbye1FW+qOEcY5S+SpY9T0NiuqvtfMY=
github.com/redis/go-redis/v9 v9.1.0/go.mod h1:urWj3He21Dj5k4TK1y59xH8Uj6ATueP8AH1cY3lZl4c=
-github.com/refraction-networking/utls v1.6.6 h1:igFsYBUJPYM8Rno9xUuDoM5GQrVEqY4llzEXOkL43Ig=
-github.com/refraction-networking/utls v1.6.6/go.mod h1:BC3O4vQzye5hqpmDTWUqi4P5DDhzJfkV1tdqtawQIH0=
+github.com/refraction-networking/utls v1.6.7 h1:zVJ7sP1dJx/WtVuITug3qYUq034cDq9B2MR1K67ULZM=
+github.com/refraction-networking/utls v1.6.7/go.mod h1:BC3O4vQzye5hqpmDTWUqi4P5DDhzJfkV1tdqtawQIH0=
github.com/remeh/sizedwaitgroup v1.0.0 h1:VNGGFwNo/R5+MJBf6yrsr110p0m4/OX4S3DCy7Kyl5E=
github.com/remeh/sizedwaitgroup v1.0.0/go.mod h1:3j2R4OIe/SeS6YDhICBy22RWjJC5eNCJ1V+9+NVNYlo=
github.com/rivo/uniseg v0.1.0/go.mod h1:J6wj4VEh+S6ZtnVlnTBMWIodfgj8LQOQFoIToxlJtxc=
From 0e881ebb78c1431afe8637d592b012d78075e304 Mon Sep 17 00:00:00 2001
From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com>
Date: Mon, 22 Jul 2024 07:20:36 +0000
Subject: [PATCH 106/283] chore(deps): bump
github.com/projectdiscovery/wappalyzergo
Bumps [github.com/projectdiscovery/wappalyzergo](https://github.com/projectdiscovery/wappalyzergo) from 0.1.10 to 0.1.11.
- [Release notes](https://github.com/projectdiscovery/wappalyzergo/releases)
- [Commits](https://github.com/projectdiscovery/wappalyzergo/compare/v0.1.10...v0.1.11)
---
updated-dependencies:
- dependency-name: github.com/projectdiscovery/wappalyzergo
dependency-type: direct:production
update-type: version-update:semver-patch
...
Signed-off-by: dependabot[bot]
---
go.mod | 2 +-
go.sum | 4 ++--
2 files changed, 3 insertions(+), 3 deletions(-)
diff --git a/go.mod b/go.mod
index c03677c803..1a6526390d 100644
--- a/go.mod
+++ b/go.mod
@@ -96,7 +96,7 @@ require (
github.com/projectdiscovery/uncover v1.0.8
github.com/projectdiscovery/useragent v0.0.60
github.com/projectdiscovery/utils v0.2.2
- github.com/projectdiscovery/wappalyzergo v0.1.10
+ github.com/projectdiscovery/wappalyzergo v0.1.11
github.com/redis/go-redis/v9 v9.1.0
github.com/seh-msft/burpxml v1.0.1
github.com/stretchr/testify v1.9.0
diff --git a/go.sum b/go.sum
index 2172188616..68aab23454 100644
--- a/go.sum
+++ b/go.sum
@@ -890,8 +890,8 @@ github.com/projectdiscovery/useragent v0.0.60 h1:qDU1rwA+XOKmSqp7yoijAN4PuvLQc2Z
github.com/projectdiscovery/useragent v0.0.60/go.mod h1:05IDiJEy2dWl3x6dnsWtJYPwT40oWha144Us7+Fwr6w=
github.com/projectdiscovery/utils v0.2.2 h1:ZzanMTiVboM7vFO6cjt+GRDtvVjZD1GChBa7knekBgs=
github.com/projectdiscovery/utils v0.2.2/go.mod h1:sZHBCrS37ejIb0/pFjKZewm1nBWMTXUsLJH3hTVww0Y=
-github.com/projectdiscovery/wappalyzergo v0.1.10 h1:cUTMw8xYXyKTAxUiGYltJxEh4OtJyCvaahj+RG3mHSY=
-github.com/projectdiscovery/wappalyzergo v0.1.10/go.mod h1:/hzgxkBFTMe2wDbA93nFfoMjULw7/vIZ9QPSAnCgUa8=
+github.com/projectdiscovery/wappalyzergo v0.1.11 h1:6RZFcu4XcZ7nxi0iSI7hfq/Ivb/eEIF8h7fnajyoibo=
+github.com/projectdiscovery/wappalyzergo v0.1.11/go.mod h1:/hzgxkBFTMe2wDbA93nFfoMjULw7/vIZ9QPSAnCgUa8=
github.com/projectdiscovery/yamldoc-go v1.0.4 h1:eZoESapnMw6WAHiVgRwNqvbJEfNHEH148uthhFbG5jE=
github.com/projectdiscovery/yamldoc-go v1.0.4/go.mod h1:8PIPRcUD55UbtQdcfFR1hpIGRWG0P7alClXNGt1TBik=
github.com/prometheus/client_golang v0.9.1/go.mod h1:7SWBe2y4D6OKWSNQJUaRYU/AaXPKyh/dDVn+NZz0KFw=
From 9f08fe21d8cc271d9b6f611facaeea80cc66b2fd Mon Sep 17 00:00:00 2001
From: fudancoder
Date: Mon, 22 Jul 2024 16:59:35 +0800
Subject: [PATCH 107/283] chore: fix some comments (#5432)
Signed-off-by: fudancoder
---
pkg/testutils/integration.go | 2 +-
pkg/types/types.go | 2 +-
pkg/utils/utils.go | 2 +-
3 files changed, 3 insertions(+), 3 deletions(-)
diff --git a/pkg/testutils/integration.go b/pkg/testutils/integration.go
index c8a67e41cb..49eda29626 100644
--- a/pkg/testutils/integration.go
+++ b/pkg/testutils/integration.go
@@ -162,7 +162,7 @@ func RunNucleiArgsAndGetErrors(debug bool, env []string, extra ...string) ([]str
return results, err
}
-// RunNucleiArgsWithEnvAndGetErrors returns a list of errors in nuclei output (ERR,WRN,FTL)
+// RunNucleiArgsWithEnvAndGetResults returns a list of results in nuclei output (ERR,WRN,FTL)
func RunNucleiArgsWithEnvAndGetResults(debug bool, env []string, extra ...string) ([]string, error) {
cmd := exec.Command("./nuclei")
extra = append(extra, ExtraDebugArgs...)
diff --git a/pkg/types/types.go b/pkg/types/types.go
index 48e9039d37..9d05a0b742 100644
--- a/pkg/types/types.go
+++ b/pkg/types/types.go
@@ -592,7 +592,7 @@ func (o *Options) GetValidAbsPath(helperFilePath, templatePath string) (string,
return "", errorutil.New("access to helper file %v denied", helperFilePath)
}
-// isRootDir checks if given is root directory
+// isHomeDir checks if given is home directory
func isHomeDir(path string) bool {
homeDir := folderutil.HomeDirOrDefault("")
return strings.HasPrefix(path, homeDir)
diff --git a/pkg/utils/utils.go b/pkg/utils/utils.go
index dfe37ae99e..9a8c669f9b 100644
--- a/pkg/utils/utils.go
+++ b/pkg/utils/utils.go
@@ -35,7 +35,7 @@ func IsURL(input string) bool {
return err == nil && u.Scheme != "" && u.Host != ""
}
-// ReadFromPathOrURL reads and returns the contents of a file or url.
+// ReaderFromPathOrURL reads and returns the contents of a file or url.
func ReaderFromPathOrURL(templatePath string, catalog catalog.Catalog) (io.ReadCloser, error) {
if IsURL(templatePath) {
resp, err := retryablehttp.DefaultClient().Get(templatePath)
From 1fa0e2b4738a762468d829b6908ef2d4a9c5c741 Mon Sep 17 00:00:00 2001
From: Dogan Can Bakir <65292895+dogancanbakir@users.noreply.github.com>
Date: Mon, 22 Jul 2024 15:55:02 +0300
Subject: [PATCH 108/283] disable http probe when passive mode is enabled
(#5418)
---
lib/config.go | 1 +
1 file changed, 1 insertion(+)
diff --git a/lib/config.go b/lib/config.go
index df44566225..7a9419aa50 100644
--- a/lib/config.go
+++ b/lib/config.go
@@ -396,6 +396,7 @@ func WithHeaders(headers []string) NucleiSDKOptions {
func EnablePassiveMode() NucleiSDKOptions {
return func(e *NucleiEngine) error {
e.opts.OfflineHTTP = true
+ e.opts.DisableHTTPProbe = true
return nil
}
}
From c5a39494734b1121f03e4d0536b276c801890961 Mon Sep 17 00:00:00 2001
From: shubo <814183583@qq.com>
Date: Tue, 23 Jul 2024 09:25:46 +0800
Subject: [PATCH 109/283] fix: Missing close statements file.Close() &
ticker.Stop()
---
internal/pdcp/writer.go | 2 +-
pkg/scan/charts/echarts.go | 1 +
pkg/scan/events/stats_build.go | 1 +
3 files changed, 3 insertions(+), 1 deletion(-)
diff --git a/internal/pdcp/writer.go b/internal/pdcp/writer.go
index 5dfda78ca2..2197041137 100644
--- a/internal/pdcp/writer.go
+++ b/internal/pdcp/writer.go
@@ -142,7 +142,7 @@ func (u *UploadWriter) autoCommit(ctx context.Context, r *io.PipeReader) {
// temporary buffer to store the results
buff := &bytes.Buffer{}
ticker := time.NewTicker(flushTimer)
-
+ defer ticker.Stop()
for {
select {
case <-ctx.Done():
diff --git a/pkg/scan/charts/echarts.go b/pkg/scan/charts/echarts.go
index a2a9815888..69960588da 100644
--- a/pkg/scan/charts/echarts.go
+++ b/pkg/scan/charts/echarts.go
@@ -30,6 +30,7 @@ func (s *ScanEventsCharts) GenerateHTML(filePath string) error {
if err != nil {
return err
}
+ defer output.Close()
return page.Render(output)
}
diff --git a/pkg/scan/events/stats_build.go b/pkg/scan/events/stats_build.go
index 0f01724411..6fe5f27174 100644
--- a/pkg/scan/events/stats_build.go
+++ b/pkg/scan/events/stats_build.go
@@ -56,6 +56,7 @@ func (s *ScanStatsWorker) initEventsFile() error {
if err != nil {
return err
}
+ defer f.Close()
s.enc = json.NewEncoder(f)
return nil
}
From f930e9a58fcd93293461e0e65a9c3c1c9ed973bb Mon Sep 17 00:00:00 2001
From: jarnpher_rice
Date: Thu, 25 Jul 2024 05:29:35 +0800
Subject: [PATCH 110/283] chore(deps): change github.com/denisenkom/go-mssqldb
to github.com/microsoft/go-mssqldb (#5419)
---
go.mod | 4 ++--
go.sum | 20 ++++++++------------
pkg/js/generated/ts/mssql.ts | 2 +-
pkg/js/libs/mssql/memo.mssql.go | 2 +-
pkg/js/libs/mssql/mssql.go | 4 ++--
5 files changed, 14 insertions(+), 18 deletions(-)
diff --git a/go.mod b/go.mod
index 1a6526390d..290bba6ee1 100644
--- a/go.mod
+++ b/go.mod
@@ -61,7 +61,6 @@ require (
github.com/cespare/xxhash v1.1.0
github.com/charmbracelet/glamour v0.6.0
github.com/clbanning/mxj/v2 v2.7.0
- github.com/denisenkom/go-mssqldb v0.12.3
github.com/ditashi/jsbeautifier-go v0.0.0-20141206144643-2520a8026a9c
github.com/docker/go-units v0.5.0
github.com/dop251/goja v0.0.0-20240220182346-e401ed450204
@@ -77,6 +76,7 @@ require (
github.com/lib/pq v1.10.9
github.com/mattn/go-sqlite3 v1.14.22
github.com/mholt/archiver v3.1.1+incompatible
+ github.com/microsoft/go-mssqldb v1.6.0
github.com/ory/dockertest/v3 v3.10.0
github.com/praetorian-inc/fingerprintx v1.1.9
github.com/projectdiscovery/dsl v0.1.6
@@ -158,7 +158,7 @@ require (
github.com/goccy/go-json v0.10.2 // indirect
github.com/gogo/protobuf v1.3.2 // indirect
github.com/golang-jwt/jwt/v5 v5.2.1 // indirect
- github.com/golang-sql/civil v0.0.0-20190719163853-cb61b32ac6fe // indirect
+ github.com/golang-sql/civil v0.0.0-20220223132316-b832511892a9 // indirect
github.com/golang-sql/sqlexp v0.1.0 // indirect
github.com/google/certificate-transparency-go v1.1.4 // indirect
github.com/google/go-github/v30 v30.1.0 // indirect
diff --git a/go.sum b/go.sum
index 68aab23454..d781abdc20 100644
--- a/go.sum
+++ b/go.sum
@@ -40,17 +40,18 @@ dario.cat/mergo v1.0.0/go.mod h1:uNxQE+84aUszobStD9th8a29P2fMDhsBdgRYvZOxGmk=
dmitri.shuralyov.com/gpu/mtl v0.0.0-20190408044501-666a987793e9/go.mod h1:H6x//7gZCb22OMCxBHrMx7a5I7Hp++hsVxbQ4BYO7hU=
git.mills.io/prologic/smtpd v0.0.0-20210710122116-a525b76c287a h1:3i+FJ7IpSZHL+VAjtpQeZCRhrpP0odl5XfoLBY4fxJ8=
git.mills.io/prologic/smtpd v0.0.0-20210710122116-a525b76c287a/go.mod h1:C7hXLmFmPYPjIDGfQl1clsmQ5TMEQfmzWTrJk475bUs=
-github.com/Azure/azure-sdk-for-go/sdk/azcore v0.19.0/go.mod h1:h6H6c8enJmmocHUbLiiGY6sx7f9i+X3m1CHdd5c6Rdw=
github.com/Azure/azure-sdk-for-go/sdk/azcore v1.11.1 h1:E+OJmp2tPvt1W+amx48v1eqbjDYsgN+RzP4q16yV5eM=
github.com/Azure/azure-sdk-for-go/sdk/azcore v1.11.1/go.mod h1:a6xsAQUZg+VsS3TJ05SRp524Hs4pZ/AeFSr5ENf0Yjo=
-github.com/Azure/azure-sdk-for-go/sdk/azidentity v0.11.0/go.mod h1:HcM1YX14R7CJcghJGOYCgdezslRSVzqwLf/q+4Y2r/0=
github.com/Azure/azure-sdk-for-go/sdk/azidentity v1.6.0 h1:U2rTu3Ef+7w9FHKIAXM6ZyqF3UOWJZ12zIm8zECAFfg=
github.com/Azure/azure-sdk-for-go/sdk/azidentity v1.6.0/go.mod h1:9kIvujWAA58nmPmWB1m23fyWic1kYZMxD9CxaWn4Qpg=
-github.com/Azure/azure-sdk-for-go/sdk/internal v0.7.0/go.mod h1:yqy467j36fJxcRV2TzfVZ1pCb5vxm4BtZPUdYWe/Xo8=
github.com/Azure/azure-sdk-for-go/sdk/internal v1.8.0 h1:jBQA3cKT4L2rWMpgE7Yt3Hwh2aUj8KXjIGLxjHeYNNo=
github.com/Azure/azure-sdk-for-go/sdk/internal v1.8.0/go.mod h1:4OG6tQ9EOP/MT0NMjDlRzWoVFxfu9rN9B2X+tlSVktg=
github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/storage/armstorage v1.2.0 h1:Ma67P/GGprNwsslzEH6+Kb8nybI8jpDTm4Wmzu2ReK8=
github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/storage/armstorage v1.2.0/go.mod h1:c+Lifp3EDEamAkPVzMooRNOK6CZjNSdEnf1A7jsI9u4=
+github.com/Azure/azure-sdk-for-go/sdk/security/keyvault/azkeys v1.0.0 h1:yfJe15aSwEQ6Oo6J+gdfdulPNoZ3TEhmbhLIoxZcA+U=
+github.com/Azure/azure-sdk-for-go/sdk/security/keyvault/azkeys v1.0.0/go.mod h1:Q28U+75mpCaSCDowNEmhIo/rmgdkqmkmzI7N6TGR4UY=
+github.com/Azure/azure-sdk-for-go/sdk/security/keyvault/internal v0.8.0 h1:T028gtTPiYt/RMUfs8nVsAL7FDQrfLlrm/NnRG/zcC4=
+github.com/Azure/azure-sdk-for-go/sdk/security/keyvault/internal v0.8.0/go.mod h1:cw4zVQgBby0Z5f2v0itn6se2dDP17nTjbZFXW5uPyHA=
github.com/Azure/azure-sdk-for-go/sdk/storage/azblob v1.1.0 h1:nVocQV40OQne5613EeLayJiRAJuKlBGy+m22qWG+WRg=
github.com/Azure/azure-sdk-for-go/sdk/storage/azblob v1.1.0/go.mod h1:7QJP7dr2wznCMeqIrhMgWGf7XpAQnVrJqDm9nvV3Cu4=
github.com/Azure/go-ansiterm v0.0.0-20230124172434-306776ec8161 h1:L/gRVlceqvL25UVaW/CKtUDjefjrs0SPonmDGUVOYP0=
@@ -260,8 +261,6 @@ github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c
github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
github.com/davidmz/go-pageant v1.0.2 h1:bPblRCh5jGU+Uptpz6LgMZGD5hJoOt7otgT454WvHn0=
github.com/davidmz/go-pageant v1.0.2/go.mod h1:P2EDDnMqIwG5Rrp05dTRITj9z2zpGcD9efWSkTNKLIE=
-github.com/denisenkom/go-mssqldb v0.12.3 h1:pBSGx9Tq67pBOTLmxNuirNTeB8Vjmf886Kx+8Y+8shw=
-github.com/denisenkom/go-mssqldb v0.12.3/go.mod h1:k0mtMFOnU+AihqFxPMiF05rtiDrorD1Vrm1KEz5hxDo=
github.com/dgrijalva/jwt-go v3.2.0+incompatible/go.mod h1:E3ru+11k8xSBh+hMPgOLZmtrrCbhqsmaPHjLKYnJCaQ=
github.com/dgryski/go-rendezvous v0.0.0-20200823014737-9f7001d12a5f h1:lO4WD4F/rVNCu3HqELle0jiPLLBs70cWOduZpkS1E78=
github.com/dgryski/go-rendezvous v0.0.0-20200823014737-9f7001d12a5f/go.mod h1:cuUVRXasLTGF7a8hSLbxyZXjz+1KgoB3wDUb6vlszIc=
@@ -274,7 +273,6 @@ github.com/dlclark/regexp2 v1.4.1-0.20201116162257-a2a8dda75c91/go.mod h1:2pZnwu
github.com/dlclark/regexp2 v1.7.0/go.mod h1:DHkYz0B9wPfa6wondMfaivmHpzrQ3v9q8cnmRbL6yW8=
github.com/dlclark/regexp2 v1.11.0 h1:G/nrcoOa7ZXlpoa/91N3X7mM3r8eIlMBBJZvsz/mxKI=
github.com/dlclark/regexp2 v1.11.0/go.mod h1:DHkYz0B9wPfa6wondMfaivmHpzrQ3v9q8cnmRbL6yW8=
-github.com/dnaeon/go-vcr v1.2.0/go.mod h1:R4UdLID7HZT3taECzJs4YgbbH6PIGXB6W/sc5OLb6RQ=
github.com/docker/cli v24.0.5+incompatible h1:WeBimjvS0eKdH4Ygx+ihVq1Q++xg36M/rMi4aXAvodc=
github.com/docker/cli v24.0.5+incompatible/go.mod h1:JLrzqnKDaYBop7H2jaqPtU4hHvMKP+vjCwu2uszcLI8=
github.com/docker/docker v24.0.9+incompatible h1:HPGzNmwfLZWdxHqK9/II92pyi1EpYKsAqcl4G0Of9v0=
@@ -420,8 +418,8 @@ github.com/golang-jwt/jwt/v4 v4.5.0 h1:7cYmW1XlMY7h7ii7UhUyChSgS5wUJEnm9uZVTGqOW
github.com/golang-jwt/jwt/v4 v4.5.0/go.mod h1:m21LjoU+eqJr34lmDMbreY2eSTRJ1cv77w39/MY0Ch0=
github.com/golang-jwt/jwt/v5 v5.2.1 h1:OuVbFODueb089Lh128TAcimifWaLhJwVflnrgM17wHk=
github.com/golang-jwt/jwt/v5 v5.2.1/go.mod h1:pqrtFR0X4osieyHYxtmOUWsAWrfe1Q5UVIyoH402zdk=
-github.com/golang-sql/civil v0.0.0-20190719163853-cb61b32ac6fe h1:lXe2qZdvpiX5WZkZR4hgp4KJVfY3nMkvmwbVkpv1rVY=
-github.com/golang-sql/civil v0.0.0-20190719163853-cb61b32ac6fe/go.mod h1:8vg3r2VgvsThLBIFL93Qb5yWzgyZWhEmBwUJWevAkK0=
+github.com/golang-sql/civil v0.0.0-20220223132316-b832511892a9 h1:au07oEsX2xN0ktxqI+Sida1w446QrXBRJ0nee3SNZlA=
+github.com/golang-sql/civil v0.0.0-20220223132316-b832511892a9/go.mod h1:8vg3r2VgvsThLBIFL93Qb5yWzgyZWhEmBwUJWevAkK0=
github.com/golang-sql/sqlexp v0.1.0 h1:ZCD6MBpcuOVfGVqsEmY5/4FtYiKz6tSyUv9LPEDei6A=
github.com/golang-sql/sqlexp v0.1.0/go.mod h1:J4ad9Vo8ZCWQ2GMrC4UCQy1JpCbwU9m3EOqtpKwwwHI=
github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q=
@@ -713,6 +711,8 @@ github.com/mholt/archiver/v3 v3.5.1/go.mod h1:e3dqJ7H78uzsRSEACH1joayhuSyhnonssn
github.com/microcosm-cc/bluemonday v1.0.21/go.mod h1:ytNkv4RrDrLJ2pqlsSI46O6IVXmZOBBD4SaJyDwwTkM=
github.com/microcosm-cc/bluemonday v1.0.26 h1:xbqSvqzQMeEHCqMi64VAs4d8uy6Mequs3rQ0k/Khz58=
github.com/microcosm-cc/bluemonday v1.0.26/go.mod h1:JyzOCs9gkyQyjs+6h10UEVSe02CGwkhd72Xdqh78TWs=
+github.com/microsoft/go-mssqldb v1.6.0 h1:mM3gYdVwEPFrlg/Dvr2DNVEgYFG7L42l+dGc67NNNpc=
+github.com/microsoft/go-mssqldb v1.6.0/go.mod h1:00mDtPbeQCRGC1HwOOR5K/gr30P1NcEG0vx6Kbv2aJU=
github.com/miekg/dns v1.0.14/go.mod h1:W1PPwlIAgtquWBMBEV9nkV9Cazfe8ScdGz/Lj7v3Nrg=
github.com/miekg/dns v1.1.35/go.mod h1:KNUDUusw/aVsxyTYZM1oqvCicbwhgbNgztCETuNZ7xM=
github.com/miekg/dns v1.1.59 h1:C9EXc/UToRwKLhK5wKU/I4QVsBUc8kE6MkHBkeypWZs=
@@ -741,7 +741,6 @@ github.com/modern-go/reflect2 v0.0.0-20180701023420-4b7aa43c6742/go.mod h1:bx2lN
github.com/modern-go/reflect2 v1.0.1/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0=
github.com/modern-go/reflect2 v1.0.2 h1:xBagoLtFs94CBntxluKeaWgTMpvLxC4ur3nMaC9Gz0M=
github.com/modern-go/reflect2 v1.0.2/go.mod h1:yWuevngMOJpCy52FWWMvUC8ws7m/LJsjYzDa0/r8luk=
-github.com/modocache/gover v0.0.0-20171022184752-b58185e213c5/go.mod h1:caMODM3PzxT8aQXRPkAt8xlV/e7d7w8GM5g0fa5F0D8=
github.com/mohae/deepcopy v0.0.0-20170929034955-c48cc78d4826 h1:RWengNIwukTxcDr9M+97sNutRR1RKhG96O6jWumTTnw=
github.com/mohae/deepcopy v0.0.0-20170929034955-c48cc78d4826/go.mod h1:TaXosZuwdSHYgviHp1DAtfrULt5eUgsSMsZf+YrPgl8=
github.com/mreiferson/go-httpclient v0.0.0-20160630210159-31f0106b4474/go.mod h1:OQA4XLvDbMgS8P0CevmM4m9Q3Jq4phKUzcocxuGJ5m8=
@@ -811,7 +810,6 @@ github.com/pierrec/lz4/v4 v4.1.21 h1:yOVMLb6qSIDP67pl/5F7RepeKYu/VmTyEXvuMI5d9mQ
github.com/pierrec/lz4/v4 v4.1.21/go.mod h1:gZWDp/Ze/IJXGXf23ltt2EXimqmTUXEy0GFuRQyBid4=
github.com/pjbgf/sha1cd v0.3.0 h1:4D5XXmUUBUl/xQ6IjCkEAbqXskkq/4O7LmGn0AqMDs4=
github.com/pjbgf/sha1cd v0.3.0/go.mod h1:nZ1rrWOcGJ5uZgEEVL1VUM9iRQiZvWdbZjkKyFzPPsI=
-github.com/pkg/browser v0.0.0-20180916011732-0a3d74bf9ce4/go.mod h1:4OwLy04Bl9Ef3GJJCoec+30X3LQs/0/m4HFRt/2LUSA=
github.com/pkg/browser v0.0.0-20240102092130-5ac0b6a4141c h1:+mdjkGKdHQG3305AYmdv1U2eRNDiU2ErMBj1gwrq8eQ=
github.com/pkg/browser v0.0.0-20240102092130-5ac0b6a4141c/go.mod h1:7rwL4CYBLnjLxUqIJNnCWiEdr3bn6IUYi15bNlnbCCU=
github.com/pkg/errors v0.8.0/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
@@ -1174,7 +1172,6 @@ golang.org/x/crypto v0.0.0-20190701094942-4def268fd1a4/go.mod h1:yigFU9vqHzYiE8U
golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
golang.org/x/crypto v0.0.0-20200510223506-06a226fb4e37/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto=
golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto=
-golang.org/x/crypto v0.0.0-20201016220609-9e8e0b390897/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto=
golang.org/x/crypto v0.0.0-20201124201722-c8d3bf9c5392/go.mod h1:jdWPYTVW3xRLrWPugEBEK3UY2ZEsg3UU495nc5E+M+I=
golang.org/x/crypto v0.0.0-20201208171446-5f87f3452ae9/go.mod h1:jdWPYTVW3xRLrWPugEBEK3UY2ZEsg3UU495nc5E+M+I=
golang.org/x/crypto v0.0.0-20210220033148-5ea612d1eb83/go.mod h1:jdWPYTVW3xRLrWPugEBEK3UY2ZEsg3UU495nc5E+M+I=
@@ -1268,7 +1265,6 @@ golang.org/x/net v0.0.0-20201110031124-69a78807bb2b/go.mod h1:sp8m0HH+o8qH0wwXwY
golang.org/x/net v0.0.0-20201209123823-ac852fbbde11/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg=
golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg=
golang.org/x/net v0.0.0-20210525063256-abc453219eb5/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y=
-golang.org/x/net v0.0.0-20210610132358-84b48f89b13b/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y=
golang.org/x/net v0.0.0-20210916014120-12bc252f5db8/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y=
golang.org/x/net v0.0.0-20211112202133-69e39bad7dc2/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y=
golang.org/x/net v0.0.0-20220127200216-cd36cc0744dd/go.mod h1:CfG3xpIq0wQ8r1q4Su4UZFWDARRcnwPjda9FqA0JpMk=
diff --git a/pkg/js/generated/ts/mssql.ts b/pkg/js/generated/ts/mssql.ts
index bff2edf8b5..947aa0e9ed 100755
--- a/pkg/js/generated/ts/mssql.ts
+++ b/pkg/js/generated/ts/mssql.ts
@@ -2,7 +2,7 @@
/**
* Client is a client for MS SQL database.
- * Internally client uses denisenkom/go-mssqldb driver.
+ * Internally client uses microsoft/go-mssqldb driver.
* @example
* ```javascript
* const mssql = require('nuclei/mssql');
diff --git a/pkg/js/libs/mssql/memo.mssql.go b/pkg/js/libs/mssql/memo.mssql.go
index eb169761a0..e57dec5cdd 100755
--- a/pkg/js/libs/mssql/memo.mssql.go
+++ b/pkg/js/libs/mssql/memo.mssql.go
@@ -5,7 +5,7 @@ import (
"errors"
"fmt"
- _ "github.com/denisenkom/go-mssqldb"
+ _ "github.com/microsoft/go-mssqldb"
"github.com/projectdiscovery/nuclei/v3/pkg/protocols/common/protocolstate"
)
diff --git a/pkg/js/libs/mssql/mssql.go b/pkg/js/libs/mssql/mssql.go
index 1b0efa4284..5660cc2a6f 100644
--- a/pkg/js/libs/mssql/mssql.go
+++ b/pkg/js/libs/mssql/mssql.go
@@ -9,14 +9,14 @@ import (
"strings"
"time"
- _ "github.com/denisenkom/go-mssqldb"
+ _ "github.com/microsoft/go-mssqldb"
"github.com/praetorian-inc/fingerprintx/pkg/plugins/services/mssql"
"github.com/projectdiscovery/nuclei/v3/pkg/protocols/common/protocolstate"
)
type (
// Client is a client for MS SQL database.
- // Internally client uses denisenkom/go-mssqldb driver.
+ // Internally client uses microsoft/go-mssqldb driver.
// @example
// ```javascript
// const mssql = require('nuclei/mssql');
From 33dbb51505c7fbcdb9ba229dafdb9e556f53ade5 Mon Sep 17 00:00:00 2001
From: Ramana Reddy <90540245+RamanaReddy0M@users.noreply.github.com>
Date: Fri, 26 Jul 2024 00:01:05 +0530
Subject: [PATCH 111/283] fix unresolved `interactsh-url` variable with fuzzing
(#5289)
* fix unresolved interactsh variable with fuzzing
* fix variables override with fuzzing
---
pkg/fuzz/execute.go | 7 +++++--
pkg/fuzz/parts.go | 4 ++--
pkg/protocols/common/interactsh/const.go | 2 +-
3 files changed, 8 insertions(+), 5 deletions(-)
diff --git a/pkg/fuzz/execute.go b/pkg/fuzz/execute.go
index 4f33ba6ddc..8eb57f6b40 100644
--- a/pkg/fuzz/execute.go
+++ b/pkg/fuzz/execute.go
@@ -167,9 +167,12 @@ func (rule *Rule) evaluateVarsWithInteractsh(data map[string]interface{}, intera
if rule.options.Interactsh != nil {
// Iterate through the data to replace and evaluate variables with Interactsh URLs
for k, v := range data {
+ value := fmt.Sprint(v)
// Replace variables with Interactsh URLs and collect new URLs
- got, oastUrls := rule.options.Interactsh.Replace(fmt.Sprint(v), interactshUrls)
-
+ got, oastUrls := rule.options.Interactsh.Replace(value, interactshUrls)
+ if got != value {
+ data[k] = got
+ }
// Append new OAST URLs if any
if len(oastUrls) > 0 {
interactshUrls = append(interactshUrls, oastUrls...)
diff --git a/pkg/fuzz/parts.go b/pkg/fuzz/parts.go
index 6ab1643296..86e1df9f93 100644
--- a/pkg/fuzz/parts.go
+++ b/pkg/fuzz/parts.go
@@ -181,9 +181,9 @@ func (rule *Rule) execWithInput(input *ExecuteRuleInput, httpReq *retryablehttp.
// for fuzzing.
func (rule *Rule) executeEvaluate(input *ExecuteRuleInput, _, value, payload string, interactshURLs []string) (string, []string) {
// TODO: Handle errors
- values := generators.MergeMaps(input.Values, map[string]interface{}{
+ values := generators.MergeMaps(rule.options.Variables.GetAll(), map[string]interface{}{
"value": value,
- }, rule.options.Options.Vars.AsMap(), rule.options.Variables.GetAll())
+ }, rule.options.Options.Vars.AsMap(), input.Values)
firstpass, _ := expressions.Evaluate(payload, values)
interactData, interactshURLs := rule.options.Interactsh.Replace(firstpass, interactshURLs)
evaluated, _ := expressions.Evaluate(interactData, values)
diff --git a/pkg/protocols/common/interactsh/const.go b/pkg/protocols/common/interactsh/const.go
index 079940240f..aad130d468 100644
--- a/pkg/protocols/common/interactsh/const.go
+++ b/pkg/protocols/common/interactsh/const.go
@@ -8,7 +8,7 @@ import (
var (
defaultInteractionDuration = 60 * time.Second
- interactshURLMarkerRegex = regexp.MustCompile(`{{interactsh-url(?:_[0-9]+){0,3}}}`)
+ interactshURLMarkerRegex = regexp.MustCompile(`(%7[B|b]|\{){2}(interactsh-url(?:_[0-9]+){0,3})(%7[D|d]|\}){2}`)
ErrInteractshClientNotInitialized = errors.New("interactsh client not initialized")
)
From 6d325a4ebe5cd7d33084e97fc74d961eac241659 Mon Sep 17 00:00:00 2001
From: Dwi Siswanto
Date: Fri, 26 Jul 2024 22:24:35 +0700
Subject: [PATCH 112/283] feat(http): assign `customHeaders` to the map
directly (#5445)
also add skip expr if header key is "Host"
Signed-off-by: Dwi Siswanto
---
pkg/protocols/http/request.go | 7 ++++++-
1 file changed, 6 insertions(+), 1 deletion(-)
diff --git a/pkg/protocols/http/request.go b/pkg/protocols/http/request.go
index 0df13b8872..2eec929808 100644
--- a/pkg/protocols/http/request.go
+++ b/pkg/protocols/http/request.go
@@ -1083,10 +1083,15 @@ func (request *Request) setCustomHeaders(req *generatedRequest) {
req.rawRequest.Headers[k] = v
} else {
kk, vv := strings.TrimSpace(k), strings.TrimSpace(v)
- req.request.Header.Set(kk, vv)
+ // NOTE(dwisiswant0): Do we really not need to convert it first into
+ // lowercase?
if kk == "Host" {
req.request.Host = vv
+
+ continue
}
+
+ req.request.Header[kk] = []string{vv}
}
}
}
From 2418319df403b74ea2e38557adb1806589e9b87c Mon Sep 17 00:00:00 2001
From: Tarun Koyalwar <45962551+tarunKoyalwar@users.noreply.github.com>
Date: Sat, 27 Jul 2024 02:46:34 +0530
Subject: [PATCH 113/283] js: generate matcher-status event (#5450)
* js: generate matcher-status event
* isPortOpen: use fastdialer instance
* update sdk unit test
* add docs :)
---
examples/advanced/advanced.go | 13 ++-
pkg/js/compiler/compiler.go | 21 ++++-
pkg/js/global/scripts.go | 26 ++++--
pkg/protocols/javascript/js.go | 150 +++++++++++++++++++--------------
4 files changed, 137 insertions(+), 73 deletions(-)
diff --git a/examples/advanced/advanced.go b/examples/advanced/advanced.go
index 110160f9a1..79355e5d9d 100644
--- a/examples/advanced/advanced.go
+++ b/examples/advanced/advanced.go
@@ -1,13 +1,24 @@
package main
import (
+ "context"
+
nuclei "github.com/projectdiscovery/nuclei/v3/lib"
+ "github.com/projectdiscovery/nuclei/v3/pkg/installer"
syncutil "github.com/projectdiscovery/utils/sync"
)
func main() {
+ ctx := context.Background()
+ // when running nuclei in parallel for first time it is a good practice to make sure
+ // templates exists first
+ tm := installer.TemplateManager{}
+ if err := tm.FreshInstallIfNotExists(); err != nil {
+ panic(err)
+ }
+
// create nuclei engine with options
- ne, err := nuclei.NewThreadSafeNucleiEngine()
+ ne, err := nuclei.NewThreadSafeNucleiEngineCtx(ctx)
if err != nil {
panic(err)
}
diff --git a/pkg/js/compiler/compiler.go b/pkg/js/compiler/compiler.go
index f50e44ff2a..99cbcce923 100644
--- a/pkg/js/compiler/compiler.go
+++ b/pkg/js/compiler/compiler.go
@@ -55,6 +55,11 @@ type ExecuteArgs struct {
TemplateCtx map[string]interface{} // templateCtx contains template scoped variables
}
+// Map returns a merged map of the TemplateCtx and Args fields.
+func (e *ExecuteArgs) Map() map[string]interface{} {
+ return generators.MergeMaps(e.TemplateCtx, e.Args)
+}
+
// NewExecuteArgs returns a new execute arguments.
func NewExecuteArgs() *ExecuteArgs {
return &ExecuteArgs{
@@ -66,12 +71,24 @@ func NewExecuteArgs() *ExecuteArgs {
// ExecuteResult is the result of executing a script.
type ExecuteResult map[string]interface{}
+// Map returns the map representation of the ExecuteResult
+func (e ExecuteResult) Map() map[string]interface{} {
+ if e == nil {
+ return make(map[string]interface{})
+ }
+ return e
+}
+
+// NewExecuteResult returns a new execute result instance
func NewExecuteResult() ExecuteResult {
return make(map[string]interface{})
}
// GetSuccess returns whether the script was successful or not.
func (e ExecuteResult) GetSuccess() bool {
+ if e == nil {
+ return false
+ }
val, ok := e["success"].(bool)
if !ok {
return false
@@ -114,7 +131,9 @@ func (c *Compiler) ExecuteWithOptions(program *goja.Program, args *ExecuteArgs,
if val, ok := err.(*goja.Exception); ok {
err = val.Unwrap()
}
- return nil, err
+ e := NewExecuteResult()
+ e["error"] = err.Error()
+ return e, err
}
var res ExecuteResult
if opts.exports != nil {
diff --git a/pkg/js/global/scripts.go b/pkg/js/global/scripts.go
index c6771fadf3..2c1d56e12b 100644
--- a/pkg/js/global/scripts.go
+++ b/pkg/js/global/scripts.go
@@ -2,6 +2,7 @@ package global
import (
"bytes"
+ "context"
"embed"
"math/rand"
"net"
@@ -12,8 +13,10 @@ import (
"github.com/logrusorgru/aurora"
"github.com/projectdiscovery/gologger"
"github.com/projectdiscovery/nuclei/v3/pkg/js/gojs"
+ "github.com/projectdiscovery/nuclei/v3/pkg/protocols/common/protocolstate"
"github.com/projectdiscovery/nuclei/v3/pkg/protocols/common/utils/vardump"
"github.com/projectdiscovery/nuclei/v3/pkg/types"
+ "github.com/projectdiscovery/utils/errkit"
errorutil "github.com/projectdiscovery/utils/errors"
stringsutil "github.com/projectdiscovery/utils/strings"
)
@@ -111,11 +114,16 @@ func initBuiltInFunc(runtime *goja.Runtime) {
},
Description: "isPortOpen checks if given TCP port is open on host. timeout is optional and defaults to 5 seconds",
FuncDecl: func(host string, port string, timeout ...int) (bool, error) {
- timeoutInSec := 5
+ ctx := context.Background()
if len(timeout) > 0 {
- timeoutInSec = timeout[0]
+ var cancel context.CancelFunc
+ ctx, cancel = context.WithTimeout(ctx, time.Duration(timeout[0])*time.Second)
+ defer cancel()
}
- conn, err := net.DialTimeout("tcp", net.JoinHostPort(host, port), time.Duration(timeoutInSec)*time.Second)
+ if host == "" || port == "" {
+ return false, errkit.New("isPortOpen: host or port is empty")
+ }
+ conn, err := protocolstate.Dialer.Dial(ctx, "tcp", net.JoinHostPort(host, port))
if err != nil {
return false, err
}
@@ -131,16 +139,20 @@ func initBuiltInFunc(runtime *goja.Runtime) {
},
Description: "isUDPPortOpen checks if the given UDP port is open on the host. Timeout is optional and defaults to 5 seconds.",
FuncDecl: func(host string, port string, timeout ...int) (bool, error) {
- timeoutInSec := 5
+ ctx := context.Background()
if len(timeout) > 0 {
- timeoutInSec = timeout[0]
+ var cancel context.CancelFunc
+ ctx, cancel = context.WithTimeout(ctx, time.Duration(timeout[0])*time.Second)
+ defer cancel()
+ }
+ if host == "" || port == "" {
+ return false, errkit.New("isPortOpen: host or port is empty")
}
- conn, err := net.DialTimeout("udp", net.JoinHostPort(host, port), time.Duration(timeoutInSec)*time.Second)
+ conn, err := protocolstate.Dialer.Dial(ctx, "udp", net.JoinHostPort(host, port))
if err != nil {
return false, err
}
_ = conn.Close()
-
return true, nil
},
})
diff --git a/pkg/protocols/javascript/js.go b/pkg/protocols/javascript/js.go
index 2e43c61cd9..fbcd1a6ff9 100644
--- a/pkg/protocols/javascript/js.go
+++ b/pkg/protocols/javascript/js.go
@@ -35,6 +35,7 @@ import (
"github.com/projectdiscovery/utils/errkit"
errorutil "github.com/projectdiscovery/utils/errors"
iputil "github.com/projectdiscovery/utils/ip"
+ mapsutil "github.com/projectdiscovery/utils/maps"
syncutil "github.com/projectdiscovery/utils/sync"
urlutil "github.com/projectdiscovery/utils/url"
)
@@ -346,17 +347,33 @@ func (request *Request) ExecuteWithResults(target *contextargs.Context, dynamicV
TimeoutVariants: requestOptions.Options.GetTimeouts(),
Source: &request.PreCondition, Context: target.Context(),
})
- if err != nil {
- return errorutil.NewWithTag(request.TemplateID, "could not execute pre-condition: %s", err)
- }
- if !result.GetSuccess() || types.ToString(result["error"]) != "" {
- gologger.Warning().Msgf("[%s] Precondition for request %s was not satisfied\n", request.TemplateID, request.PreCondition)
- request.options.Progress.IncrementFailedRequestsBy(1)
- return nil
- }
- if request.options.Options.Debug || request.options.Options.DebugRequests {
- request.options.Progress.IncrementRequests()
- gologger.Debug().Msgf("[%s] Precondition for request was satisfied\n", request.TemplateID)
+ // if precondition was successful
+ if err == nil && result.GetSuccess() {
+ if request.options.Options.Debug || request.options.Options.DebugRequests {
+ request.options.Progress.IncrementRequests()
+ gologger.Debug().Msgf("[%s] Precondition for request was satisfied\n", request.TemplateID)
+ }
+ } else {
+ var outError error
+ // if js code failed to execute
+ if err != nil {
+ outError = errkit.Append(errkit.New("pre-condition not satisfied skipping template execution"), err)
+ } else {
+ // execution successful but pre-condition returned false
+ outError = errkit.New("pre-condition not satisfied skipping template execution")
+ }
+ results := map[string]interface{}(result)
+ results["error"] = outError.Error()
+ // generate and return failed event
+ data := request.generateEventData(input, results, hostPort)
+ data = generators.MergeMaps(data, payloadValues)
+ event := eventcreator.CreateEventWithAdditionalOptions(request, data, request.options.Options.Debug || request.options.Options.DebugResponse, func(wrappedEvent *output.InternalWrappedEvent) {
+ allVars := argsCopy.Map()
+ allVars = generators.MergeMaps(allVars, data)
+ wrappedEvent.OperatorsResult.PayloadValues = allVars
+ })
+ callback(event)
+ return err
}
}
@@ -531,24 +548,72 @@ func (request *Request) executeRequestWithPayloads(hostPort string, input *conte
}
}
+ values := mapsutil.Merge(payloadValues, results)
+ // generate event data
+ data := request.generateEventData(input, values, hostPort)
+
+ // add and get values from templatectx
+ request.options.AddTemplateVars(input.MetaInput, request.Type(), request.GetID(), data)
+ data = generators.MergeMaps(data, request.options.GetTemplateCtx(input.MetaInput).GetAll())
+
+ if requestOptions.Options.Debug || requestOptions.Options.DebugRequests || requestOptions.Options.StoreResponse {
+ msg := fmt.Sprintf("[%s] Dumped Javascript response for %s:\n%v", requestOptions.TemplateID, input.MetaInput.Input, vardump.DumpVariables(results))
+ if requestOptions.Options.Debug || requestOptions.Options.DebugRequests {
+ gologger.Debug().Str("address", input.MetaInput.Input).Msg(msg)
+ }
+ if requestOptions.Options.StoreResponse {
+ request.options.Output.WriteStoreDebugData(input.MetaInput.Input, request.options.TemplateID, request.Type().String(), msg)
+ }
+ }
+
+ if _, ok := data["error"]; ok {
+ event := eventcreator.CreateEventWithAdditionalOptions(request, generators.MergeMaps(data, payloadValues), request.options.Options.Debug || request.options.Options.DebugResponse, func(wrappedEvent *output.InternalWrappedEvent) {
+ wrappedEvent.OperatorsResult.PayloadValues = payload
+ })
+ callback(event)
+ return err
+ }
+
+ if request.options.Interactsh != nil {
+ request.options.Interactsh.MakePlaceholders(interactshURLs, data)
+ }
+
+ var event *output.InternalWrappedEvent
+ if len(interactshURLs) == 0 {
+ event = eventcreator.CreateEventWithAdditionalOptions(request, generators.MergeMaps(data, payloadValues), request.options.Options.Debug || request.options.Options.DebugResponse, func(wrappedEvent *output.InternalWrappedEvent) {
+ wrappedEvent.OperatorsResult.PayloadValues = payload
+ })
+ callback(event)
+ } else if request.options.Interactsh != nil {
+ event = &output.InternalWrappedEvent{InternalEvent: data, UsesInteractsh: true}
+ request.options.Interactsh.RequestEvent(interactshURLs, &interactsh.RequestData{
+ MakeResultFunc: request.MakeResultEvent,
+ Event: event,
+ Operators: request.CompiledOperators,
+ MatchFunc: request.Match,
+ ExtractFunc: request.Extract,
+ })
+ }
+ return nil
+}
+
+// generateEventData generates event data for the request
+func (request *Request) generateEventData(input *contextargs.Context, values map[string]interface{}, matched string) map[string]interface{} {
data := make(map[string]interface{})
- for k, v := range payloadValues {
+ for k, v := range values {
data[k] = v
}
data["type"] = request.Type().String()
- for k, v := range results {
- data[k] = v
- }
+ data["request-pre-condition"] = beautifyJavascript(request.PreCondition)
data["request"] = beautifyJavascript(request.Code)
data["host"] = input.MetaInput.Input
- data["matched"] = hostPort
- data["template-path"] = requestOptions.TemplatePath
- data["template-id"] = requestOptions.TemplateID
- data["template-info"] = requestOptions.TemplateInfo
+ data["matched"] = matched
+ data["template-path"] = request.options.TemplatePath
+ data["template-id"] = request.options.TemplateID
+ data["template-info"] = request.options.TemplateInfo
if request.StopAtFirstMatch || request.options.StopAtFirstMatch {
data["stop-at-first-match"] = true
}
-
// add ip address to data
if input.MetaInput.CustomIP != "" {
data["ip"] = input.MetaInput.CustomIP
@@ -588,50 +653,7 @@ func (request *Request) executeRequestWithPayloads(hostPort string, input *conte
}
}
}
-
- // add and get values from templatectx
- request.options.AddTemplateVars(input.MetaInput, request.Type(), request.GetID(), data)
- data = generators.MergeMaps(data, request.options.GetTemplateCtx(input.MetaInput).GetAll())
-
- if requestOptions.Options.Debug || requestOptions.Options.DebugRequests || requestOptions.Options.StoreResponse {
- msg := fmt.Sprintf("[%s] Dumped Javascript response for %s:\n%v", requestOptions.TemplateID, input.MetaInput.Input, vardump.DumpVariables(results))
- if requestOptions.Options.Debug || requestOptions.Options.DebugRequests {
- gologger.Debug().Str("address", input.MetaInput.Input).Msg(msg)
- }
- if requestOptions.Options.StoreResponse {
- request.options.Output.WriteStoreDebugData(input.MetaInput.Input, request.options.TemplateID, request.Type().String(), msg)
- }
- }
-
- if _, ok := data["error"]; ok {
- event := eventcreator.CreateEventWithAdditionalOptions(request, generators.MergeMaps(data, payloadValues), request.options.Options.Debug || request.options.Options.DebugResponse, func(wrappedEvent *output.InternalWrappedEvent) {
- wrappedEvent.OperatorsResult.PayloadValues = payload
- })
- callback(event)
- return err
- }
-
- if request.options.Interactsh != nil {
- request.options.Interactsh.MakePlaceholders(interactshURLs, data)
- }
-
- var event *output.InternalWrappedEvent
- if len(interactshURLs) == 0 {
- event = eventcreator.CreateEventWithAdditionalOptions(request, generators.MergeMaps(data, payloadValues), request.options.Options.Debug || request.options.Options.DebugResponse, func(wrappedEvent *output.InternalWrappedEvent) {
- wrappedEvent.OperatorsResult.PayloadValues = payload
- })
- callback(event)
- } else if request.options.Interactsh != nil {
- event = &output.InternalWrappedEvent{InternalEvent: data, UsesInteractsh: true}
- request.options.Interactsh.RequestEvent(interactshURLs, &interactsh.RequestData{
- MakeResultFunc: request.MakeResultEvent,
- Event: event,
- Operators: request.CompiledOperators,
- MatchFunc: request.Match,
- ExtractFunc: request.Extract,
- })
- }
- return nil
+ return data
}
func (request *Request) getArgsCopy(input *contextargs.Context, payloadValues map[string]interface{}, requestOptions *protocols.ExecutorOptions, ignoreErrors bool) (*compiler.ExecuteArgs, error) {
From c67a579ab12d380d67673d25e0e0380b4e7730ea Mon Sep 17 00:00:00 2001
From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com>
Date: Mon, 29 Jul 2024 05:59:04 +0000
Subject: [PATCH 114/283] chore(deps): bump
github.com/projectdiscovery/ratelimit
Bumps [github.com/projectdiscovery/ratelimit](https://github.com/projectdiscovery/ratelimit) from 0.0.45 to 0.0.49.
- [Release notes](https://github.com/projectdiscovery/ratelimit/releases)
- [Commits](https://github.com/projectdiscovery/ratelimit/compare/v0.0.45...v0.0.49)
---
updated-dependencies:
- dependency-name: github.com/projectdiscovery/ratelimit
dependency-type: direct:production
update-type: version-update:semver-patch
...
Signed-off-by: dependabot[bot]
---
go.mod | 2 +-
go.sum | 4 ++--
2 files changed, 3 insertions(+), 3 deletions(-)
diff --git a/go.mod b/go.mod
index 290bba6ee1..1916fa0cf3 100644
--- a/go.mod
+++ b/go.mod
@@ -89,7 +89,7 @@ require (
github.com/projectdiscovery/httpx v1.6.6
github.com/projectdiscovery/mapcidr v1.1.34
github.com/projectdiscovery/n3iwf v0.0.0-20230523120440-b8cd232ff1f5
- github.com/projectdiscovery/ratelimit v0.0.45
+ github.com/projectdiscovery/ratelimit v0.0.49
github.com/projectdiscovery/rdap v0.9.1-0.20221108103045-9865884d1917
github.com/projectdiscovery/sarif v0.0.1
github.com/projectdiscovery/tlsx v1.1.6
diff --git a/go.sum b/go.sum
index d781abdc20..8fc8094c20 100644
--- a/go.sum
+++ b/go.sum
@@ -866,8 +866,8 @@ github.com/projectdiscovery/n3iwf v0.0.0-20230523120440-b8cd232ff1f5 h1:L/e8z8yw
github.com/projectdiscovery/n3iwf v0.0.0-20230523120440-b8cd232ff1f5/go.mod h1:pGW2ncnTxTxHtP9wzcIJAB+3/NMp6IiuQWd2NK7K+oc=
github.com/projectdiscovery/networkpolicy v0.0.9 h1:IrlDoYZagNNO8y+7iZeHT8k5izE+nek7TdtvEBwCxqk=
github.com/projectdiscovery/networkpolicy v0.0.9/go.mod h1:XFJ2Lnv8BE/ziQCFjBHMsH1w6VmkPiQtk+NlBpdMU7M=
-github.com/projectdiscovery/ratelimit v0.0.45 h1:h28oF+hJ0CHcdBZozT1Go7ppWmzTxSXDKNNh2G1Ot9Q=
-github.com/projectdiscovery/ratelimit v0.0.45/go.mod h1:1vSJUseDS7SjNwIBi9wNRcgsMKNTLxy/GfdlLFVbgI4=
+github.com/projectdiscovery/ratelimit v0.0.49 h1:PYatMp8g5OuoFsZOA90e48nLd2vB6a4Tw0FZ8h9zqkQ=
+github.com/projectdiscovery/ratelimit v0.0.49/go.mod h1:Xi0LTMHg4HQlmCZFzRBIhRW6N+QW5RxQ8V/Qs+Vta4k=
github.com/projectdiscovery/rawhttp v0.1.57 h1:2vCT2i1NSZbTBH+uUBrxOJjxDPKgIl2q6BGtQjs/Hko=
github.com/projectdiscovery/rawhttp v0.1.57/go.mod h1:qtthyaU0k8eqcEdza1R/fTqwyxSK4BZ511ThxgkiQtE=
github.com/projectdiscovery/rdap v0.9.1-0.20221108103045-9865884d1917 h1:m03X4gBVSorSzvmm0bFa7gDV4QNSOWPL/fgZ4kTXBxk=
From 24f4ad545a3f98d30c57948a19414f83001790ea Mon Sep 17 00:00:00 2001
From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com>
Date: Mon, 29 Jul 2024 05:59:09 +0000
Subject: [PATCH 115/283] chore(deps): bump github.com/projectdiscovery/utils
from 0.2.2 to 0.2.3
Bumps [github.com/projectdiscovery/utils](https://github.com/projectdiscovery/utils) from 0.2.2 to 0.2.3.
- [Release notes](https://github.com/projectdiscovery/utils/releases)
- [Changelog](https://github.com/projectdiscovery/utils/blob/main/CHANGELOG.md)
- [Commits](https://github.com/projectdiscovery/utils/compare/v0.2.2...v0.2.3)
---
updated-dependencies:
- dependency-name: github.com/projectdiscovery/utils
dependency-type: direct:production
update-type: version-update:semver-patch
...
Signed-off-by: dependabot[bot]
---
go.mod | 8 ++++----
go.sum | 16 ++++++++--------
2 files changed, 12 insertions(+), 12 deletions(-)
diff --git a/go.mod b/go.mod
index 290bba6ee1..32b2d891d7 100644
--- a/go.mod
+++ b/go.mod
@@ -20,12 +20,12 @@ require (
github.com/olekukonko/tablewriter v0.0.5
github.com/pkg/errors v0.9.1
github.com/projectdiscovery/clistats v0.0.20
- github.com/projectdiscovery/fastdialer v0.2.0
+ github.com/projectdiscovery/fastdialer v0.2.1
github.com/projectdiscovery/hmap v0.0.51
github.com/projectdiscovery/interactsh v1.2.0
github.com/projectdiscovery/rawhttp v0.1.57
github.com/projectdiscovery/retryabledns v1.0.68
- github.com/projectdiscovery/retryablehttp-go v1.0.69
+ github.com/projectdiscovery/retryablehttp-go v1.0.70
github.com/projectdiscovery/yamldoc-go v1.0.4
github.com/remeh/sizedwaitgroup v1.0.0
github.com/rs/xid v1.5.0
@@ -83,7 +83,7 @@ require (
github.com/projectdiscovery/fasttemplate v0.0.2
github.com/projectdiscovery/go-smb2 v0.0.0-20240129202741-052cc450c6cb
github.com/projectdiscovery/goflags v0.1.59
- github.com/projectdiscovery/gologger v1.1.15
+ github.com/projectdiscovery/gologger v1.1.16
github.com/projectdiscovery/gostruct v0.0.2
github.com/projectdiscovery/gozero v0.0.2
github.com/projectdiscovery/httpx v1.6.6
@@ -95,7 +95,7 @@ require (
github.com/projectdiscovery/tlsx v1.1.6
github.com/projectdiscovery/uncover v1.0.8
github.com/projectdiscovery/useragent v0.0.60
- github.com/projectdiscovery/utils v0.2.2
+ github.com/projectdiscovery/utils v0.2.3
github.com/projectdiscovery/wappalyzergo v0.1.11
github.com/redis/go-redis/v9 v9.1.0
github.com/seh-msft/burpxml v1.0.1
diff --git a/go.sum b/go.sum
index d781abdc20..0b97fa2d08 100644
--- a/go.sum
+++ b/go.sum
@@ -834,8 +834,8 @@ github.com/projectdiscovery/clistats v0.0.20 h1:5jO5SLiRJ7f0nDV0ndBNmBeesbROouPo
github.com/projectdiscovery/clistats v0.0.20/go.mod h1:GJ2av0KnOvK0AISQnP8hyDclYIji1LVkx2l0pwnzAu4=
github.com/projectdiscovery/dsl v0.1.6 h1:6TUYMwbjcYJ5OoRoawPX1SYD7wgubX/1FreC8SwFqgc=
github.com/projectdiscovery/dsl v0.1.6/go.mod h1:1PAV9A6X+fdyMXW0SY4i0BHrO8yN2+FB2jp25+0AT04=
-github.com/projectdiscovery/fastdialer v0.2.0 h1:ZEYKA9L5VerrD9LcXH/gtTqUHiBQvTudrbqkZSRMpJo=
-github.com/projectdiscovery/fastdialer v0.2.0/go.mod h1:bwBv51HzxK7DkCjB1EHQJKDzXKGPDLp6MgK8PPjaW/w=
+github.com/projectdiscovery/fastdialer v0.2.1 h1:or3QuGW1jlZKi+IRkwxShSAG/hgR+yamd52RqjaZ28Q=
+github.com/projectdiscovery/fastdialer v0.2.1/go.mod h1:FGPJZIPzAfR7SyDCPTsftaf61lGOqIjrJpwo2IgkNpg=
github.com/projectdiscovery/fasttemplate v0.0.2 h1:h2cISk5xDhlJEinlBQS6RRx0vOlOirB2y3Yu4PJzpiA=
github.com/projectdiscovery/fasttemplate v0.0.2/go.mod h1:XYWWVMxnItd+r0GbjA1GCsUopMw1/XusuQxdyAIHMCw=
github.com/projectdiscovery/freeport v0.0.5 h1:jnd3Oqsl4S8n0KuFkE5Hm8WGDP24ITBvmyw5pFTHS8Q=
@@ -844,8 +844,8 @@ github.com/projectdiscovery/go-smb2 v0.0.0-20240129202741-052cc450c6cb h1:rutG90
github.com/projectdiscovery/go-smb2 v0.0.0-20240129202741-052cc450c6cb/go.mod h1:FLjF1DmZ+POoGEiIQdWuYVwS++C/GwpX8YaCsTSm1RY=
github.com/projectdiscovery/goflags v0.1.59 h1:rFScpC57L0Ln0dJmBtstyxHUH7K9H4UmHACXcSkdqqk=
github.com/projectdiscovery/goflags v0.1.59/go.mod h1:dj67QGp/D81WRLtzU0HsWR20zgoGZ0cnk3Wbt9xJcuo=
-github.com/projectdiscovery/gologger v1.1.15 h1:EgXC9uQkYb5qWcazV6mH0mev23MZ0GLiGyl203nDYbU=
-github.com/projectdiscovery/gologger v1.1.15/go.mod h1:qPDCXZIxqlaKYhX0iyXZtsbtTFU152knbNRF4Z6fXUQ=
+github.com/projectdiscovery/gologger v1.1.16 h1:NsYQVNt1H8O3Wmag4sXxQxvbLbIP16bjCm/cnCvf5hc=
+github.com/projectdiscovery/gologger v1.1.16/go.mod h1:WlyfroigIqU/in8A3fTEeMJ6t5NfbCG+rgWcvI5dQiQ=
github.com/projectdiscovery/gostruct v0.0.2 h1:s8gP8ApugGM4go1pA+sVlPDXaWqNP5BBDDSv7VEdG1M=
github.com/projectdiscovery/gostruct v0.0.2/go.mod h1:H86peL4HKwMXcQQtEa6lmC8FuD9XFt6gkNR0B/Mu5PE=
github.com/projectdiscovery/gozero v0.0.2 h1:8fJeaCjxL9tpm33uG/RsCQs6HGM/NE6eA3cjkilRQ+E=
@@ -874,8 +874,8 @@ github.com/projectdiscovery/rdap v0.9.1-0.20221108103045-9865884d1917 h1:m03X4gB
github.com/projectdiscovery/rdap v0.9.1-0.20221108103045-9865884d1917/go.mod h1:JxXtZC9e195awe7EynrcnBJmFoad/BNDzW9mzFkK8Sg=
github.com/projectdiscovery/retryabledns v1.0.68 h1:EWWG7WsGTT0YvwIjHclIWSWgv4R29xMWShR6Yt5Z+pA=
github.com/projectdiscovery/retryabledns v1.0.68/go.mod h1:72W9RwsHVRIGmtc4W6i6izVtYzKBTdnCE1VciqYM5Eg=
-github.com/projectdiscovery/retryablehttp-go v1.0.69 h1:EbbHtZS1g/aT7sM3ZFNMcvnMfe9wuqXX2rEG1vGFKe4=
-github.com/projectdiscovery/retryablehttp-go v1.0.69/go.mod h1:STJ0rpHJHofbAq22yI8nZCKTVWOk50xZ+oMJHL1rc5g=
+github.com/projectdiscovery/retryablehttp-go v1.0.70 h1:J19y9kM3SL9TlYRtMD1H8tx0tGMTWBs80aFUZlICnRI=
+github.com/projectdiscovery/retryablehttp-go v1.0.70/go.mod h1:54vRm5DSwGBbBXfsjKbFDXrr7JLefWkp0iBV9mbhdoA=
github.com/projectdiscovery/sarif v0.0.1 h1:C2Tyj0SGOKbCLgHrx83vaE6YkzXEVrMXYRGLkKCr/us=
github.com/projectdiscovery/sarif v0.0.1/go.mod h1:cEYlDu8amcPf6b9dSakcz2nNnJsoz4aR6peERwV+wuQ=
github.com/projectdiscovery/stringsutil v0.0.2 h1:uzmw3IVLJSMW1kEg8eCStG/cGbYYZAja8BH3LqqJXMA=
@@ -886,8 +886,8 @@ github.com/projectdiscovery/uncover v1.0.8 h1:nE8bYJuwhqk0BEMRQRhNbjPGMy40A9gkiM
github.com/projectdiscovery/uncover v1.0.8/go.mod h1:1SwsNqjwMkJEzJQ7lQr5AHIdYd+BQlyqhO5IwIYmIAM=
github.com/projectdiscovery/useragent v0.0.60 h1:qDU1rwA+XOKmSqp7yoijAN4PuvLQc2ZvuaoWH7QIkuc=
github.com/projectdiscovery/useragent v0.0.60/go.mod h1:05IDiJEy2dWl3x6dnsWtJYPwT40oWha144Us7+Fwr6w=
-github.com/projectdiscovery/utils v0.2.2 h1:ZzanMTiVboM7vFO6cjt+GRDtvVjZD1GChBa7knekBgs=
-github.com/projectdiscovery/utils v0.2.2/go.mod h1:sZHBCrS37ejIb0/pFjKZewm1nBWMTXUsLJH3hTVww0Y=
+github.com/projectdiscovery/utils v0.2.3 h1:rkambl0EoTF/y6DpjCfSwcVUFdkAeVOtYkK3lX6InCY=
+github.com/projectdiscovery/utils v0.2.3/go.mod h1:eGuuQ5Acekg47WsFS1Q9Qxw8+vI6IxwqIQSAplBBG0c=
github.com/projectdiscovery/wappalyzergo v0.1.11 h1:6RZFcu4XcZ7nxi0iSI7hfq/Ivb/eEIF8h7fnajyoibo=
github.com/projectdiscovery/wappalyzergo v0.1.11/go.mod h1:/hzgxkBFTMe2wDbA93nFfoMjULw7/vIZ9QPSAnCgUa8=
github.com/projectdiscovery/yamldoc-go v1.0.4 h1:eZoESapnMw6WAHiVgRwNqvbJEfNHEH148uthhFbG5jE=
From ec0a8ba3ab8f69beebf16dbbaabbfe0e7fc2d9e4 Mon Sep 17 00:00:00 2001
From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com>
Date: Mon, 29 Jul 2024 06:27:15 +0000
Subject: [PATCH 116/283] chore(deps): bump github.com/projectdiscovery/dsl
from 0.1.6 to 0.1.8
Bumps [github.com/projectdiscovery/dsl](https://github.com/projectdiscovery/dsl) from 0.1.6 to 0.1.8.
- [Release notes](https://github.com/projectdiscovery/dsl/releases)
- [Commits](https://github.com/projectdiscovery/dsl/compare/v0.1.6...v0.1.8)
---
updated-dependencies:
- dependency-name: github.com/projectdiscovery/dsl
dependency-type: direct:production
update-type: version-update:semver-patch
...
Signed-off-by: dependabot[bot]
---
go.mod | 2 +-
go.sum | 4 ++--
2 files changed, 3 insertions(+), 3 deletions(-)
diff --git a/go.mod b/go.mod
index 7f4569171e..cd3ca1bbaa 100644
--- a/go.mod
+++ b/go.mod
@@ -79,7 +79,7 @@ require (
github.com/microsoft/go-mssqldb v1.6.0
github.com/ory/dockertest/v3 v3.10.0
github.com/praetorian-inc/fingerprintx v1.1.9
- github.com/projectdiscovery/dsl v0.1.6
+ github.com/projectdiscovery/dsl v0.1.8
github.com/projectdiscovery/fasttemplate v0.0.2
github.com/projectdiscovery/go-smb2 v0.0.0-20240129202741-052cc450c6cb
github.com/projectdiscovery/goflags v0.1.59
diff --git a/go.sum b/go.sum
index 37ff2cd549..fcdf1c73ce 100644
--- a/go.sum
+++ b/go.sum
@@ -832,8 +832,8 @@ github.com/projectdiscovery/cdncheck v1.1.0 h1:qDITidmJsejzpk3rMkauCh6sjI2GH9hW/
github.com/projectdiscovery/cdncheck v1.1.0/go.mod h1:sZ8U4MjHSsyaTVjBbYWHT1cwUVvUYwDX1W+WvWRicIc=
github.com/projectdiscovery/clistats v0.0.20 h1:5jO5SLiRJ7f0nDV0ndBNmBeesbROouPooH+DGMgoWq4=
github.com/projectdiscovery/clistats v0.0.20/go.mod h1:GJ2av0KnOvK0AISQnP8hyDclYIji1LVkx2l0pwnzAu4=
-github.com/projectdiscovery/dsl v0.1.6 h1:6TUYMwbjcYJ5OoRoawPX1SYD7wgubX/1FreC8SwFqgc=
-github.com/projectdiscovery/dsl v0.1.6/go.mod h1:1PAV9A6X+fdyMXW0SY4i0BHrO8yN2+FB2jp25+0AT04=
+github.com/projectdiscovery/dsl v0.1.8 h1:ulz+o097XsYgqP4QXaAQhVSkoeD2ZPWd29wX3CTodUA=
+github.com/projectdiscovery/dsl v0.1.8/go.mod h1:AYJS2WQ/q0smr2v4pEJTg4DPe6k56KFKR7UFXvzNz/4=
github.com/projectdiscovery/fastdialer v0.2.1 h1:or3QuGW1jlZKi+IRkwxShSAG/hgR+yamd52RqjaZ28Q=
github.com/projectdiscovery/fastdialer v0.2.1/go.mod h1:FGPJZIPzAfR7SyDCPTsftaf61lGOqIjrJpwo2IgkNpg=
github.com/projectdiscovery/fasttemplate v0.0.2 h1:h2cISk5xDhlJEinlBQS6RRx0vOlOirB2y3Yu4PJzpiA=
From 372eab80c872652f91c5de6883046df59bba9677 Mon Sep 17 00:00:00 2001
From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com>
Date: Mon, 29 Jul 2024 06:27:18 +0000
Subject: [PATCH 117/283] chore(deps): bump github.com/projectdiscovery/uncover
Bumps [github.com/projectdiscovery/uncover](https://github.com/projectdiscovery/uncover) from 1.0.8 to 1.0.9.
- [Release notes](https://github.com/projectdiscovery/uncover/releases)
- [Changelog](https://github.com/projectdiscovery/uncover/blob/main/.goreleaser.yml)
- [Commits](https://github.com/projectdiscovery/uncover/compare/v1.0.8...v1.0.9)
---
updated-dependencies:
- dependency-name: github.com/projectdiscovery/uncover
dependency-type: direct:production
update-type: version-update:semver-patch
...
Signed-off-by: dependabot[bot]
---
go.mod | 2 +-
go.sum | 4 ++--
2 files changed, 3 insertions(+), 3 deletions(-)
diff --git a/go.mod b/go.mod
index 7f4569171e..75fac45682 100644
--- a/go.mod
+++ b/go.mod
@@ -93,7 +93,7 @@ require (
github.com/projectdiscovery/rdap v0.9.1-0.20221108103045-9865884d1917
github.com/projectdiscovery/sarif v0.0.1
github.com/projectdiscovery/tlsx v1.1.6
- github.com/projectdiscovery/uncover v1.0.8
+ github.com/projectdiscovery/uncover v1.0.9
github.com/projectdiscovery/useragent v0.0.60
github.com/projectdiscovery/utils v0.2.3
github.com/projectdiscovery/wappalyzergo v0.1.11
diff --git a/go.sum b/go.sum
index 37ff2cd549..83ff631650 100644
--- a/go.sum
+++ b/go.sum
@@ -882,8 +882,8 @@ github.com/projectdiscovery/stringsutil v0.0.2 h1:uzmw3IVLJSMW1kEg8eCStG/cGbYYZA
github.com/projectdiscovery/stringsutil v0.0.2/go.mod h1:EJ3w6bC5fBYjVou6ryzodQq37D5c6qbAYQpGmAy+DC0=
github.com/projectdiscovery/tlsx v1.1.6 h1:iw2zwKbd2+kRQ8J1G4dLmS0CLyemd/tKz1UzcNsC77A=
github.com/projectdiscovery/tlsx v1.1.6/go.mod h1:s7SRRFdrwIZBK/RXXZi4CR/CubqFSvp8h5Bk1srEZIo=
-github.com/projectdiscovery/uncover v1.0.8 h1:nE8bYJuwhqk0BEMRQRhNbjPGMy40A9gkiMk5xq6U4u0=
-github.com/projectdiscovery/uncover v1.0.8/go.mod h1:1SwsNqjwMkJEzJQ7lQr5AHIdYd+BQlyqhO5IwIYmIAM=
+github.com/projectdiscovery/uncover v1.0.9 h1:s5RbkD/V4r8QcPkys4gTTqMuRSgXq0JprejqLSopN9Y=
+github.com/projectdiscovery/uncover v1.0.9/go.mod h1:2PUF3SpB5QNIJ8epaB2xbRzkPaxEAWRDm3Ir2ijt81U=
github.com/projectdiscovery/useragent v0.0.60 h1:qDU1rwA+XOKmSqp7yoijAN4PuvLQc2ZvuaoWH7QIkuc=
github.com/projectdiscovery/useragent v0.0.60/go.mod h1:05IDiJEy2dWl3x6dnsWtJYPwT40oWha144Us7+Fwr6w=
github.com/projectdiscovery/utils v0.2.3 h1:rkambl0EoTF/y6DpjCfSwcVUFdkAeVOtYkK3lX6InCY=
From 82e6f0c1c28f76b24e457bf50b4081209883853d Mon Sep 17 00:00:00 2001
From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com>
Date: Mon, 29 Jul 2024 06:27:19 +0000
Subject: [PATCH 118/283] chore(deps): bump
github.com/projectdiscovery/retryablehttp-go
Bumps [github.com/projectdiscovery/retryablehttp-go](https://github.com/projectdiscovery/retryablehttp-go) from 1.0.69 to 1.0.71.
- [Release notes](https://github.com/projectdiscovery/retryablehttp-go/releases)
- [Commits](https://github.com/projectdiscovery/retryablehttp-go/compare/v1.0.69...v1.0.71)
---
updated-dependencies:
- dependency-name: github.com/projectdiscovery/retryablehttp-go
dependency-type: direct:production
update-type: version-update:semver-patch
...
Signed-off-by: dependabot[bot]
---
go.mod | 2 +-
go.sum | 4 ++--
2 files changed, 3 insertions(+), 3 deletions(-)
diff --git a/go.mod b/go.mod
index 7f4569171e..145366b28c 100644
--- a/go.mod
+++ b/go.mod
@@ -25,7 +25,7 @@ require (
github.com/projectdiscovery/interactsh v1.2.0
github.com/projectdiscovery/rawhttp v0.1.57
github.com/projectdiscovery/retryabledns v1.0.68
- github.com/projectdiscovery/retryablehttp-go v1.0.70
+ github.com/projectdiscovery/retryablehttp-go v1.0.71
github.com/projectdiscovery/yamldoc-go v1.0.4
github.com/remeh/sizedwaitgroup v1.0.0
github.com/rs/xid v1.5.0
diff --git a/go.sum b/go.sum
index 37ff2cd549..8a02201d2a 100644
--- a/go.sum
+++ b/go.sum
@@ -874,8 +874,8 @@ github.com/projectdiscovery/rdap v0.9.1-0.20221108103045-9865884d1917 h1:m03X4gB
github.com/projectdiscovery/rdap v0.9.1-0.20221108103045-9865884d1917/go.mod h1:JxXtZC9e195awe7EynrcnBJmFoad/BNDzW9mzFkK8Sg=
github.com/projectdiscovery/retryabledns v1.0.68 h1:EWWG7WsGTT0YvwIjHclIWSWgv4R29xMWShR6Yt5Z+pA=
github.com/projectdiscovery/retryabledns v1.0.68/go.mod h1:72W9RwsHVRIGmtc4W6i6izVtYzKBTdnCE1VciqYM5Eg=
-github.com/projectdiscovery/retryablehttp-go v1.0.70 h1:J19y9kM3SL9TlYRtMD1H8tx0tGMTWBs80aFUZlICnRI=
-github.com/projectdiscovery/retryablehttp-go v1.0.70/go.mod h1:54vRm5DSwGBbBXfsjKbFDXrr7JLefWkp0iBV9mbhdoA=
+github.com/projectdiscovery/retryablehttp-go v1.0.71 h1:yXPNShCOwoTz7dBSJsBhBh4g4ujX62XS/BrH/fL1VyE=
+github.com/projectdiscovery/retryablehttp-go v1.0.71/go.mod h1:wY3T89EwcCKAw6iyMDvwzGPyL3d8TaBU80hnDErEKgM=
github.com/projectdiscovery/sarif v0.0.1 h1:C2Tyj0SGOKbCLgHrx83vaE6YkzXEVrMXYRGLkKCr/us=
github.com/projectdiscovery/sarif v0.0.1/go.mod h1:cEYlDu8amcPf6b9dSakcz2nNnJsoz4aR6peERwV+wuQ=
github.com/projectdiscovery/stringsutil v0.0.2 h1:uzmw3IVLJSMW1kEg8eCStG/cGbYYZAja8BH3LqqJXMA=
From 38e130201ed7f95801869666b18960d31dfbea4a Mon Sep 17 00:00:00 2001
From: Tarun Koyalwar <45962551+tarunKoyalwar@users.noreply.github.com>
Date: Wed, 31 Jul 2024 17:34:56 +0530
Subject: [PATCH 119/283] fix possible nil panic (#5473)
---
pkg/js/compiler/compiler.go | 4 +++-
1 file changed, 3 insertions(+), 1 deletion(-)
diff --git a/pkg/js/compiler/compiler.go b/pkg/js/compiler/compiler.go
index 99cbcce923..c2dc15d2cb 100644
--- a/pkg/js/compiler/compiler.go
+++ b/pkg/js/compiler/compiler.go
@@ -129,7 +129,9 @@ func (c *Compiler) ExecuteWithOptions(program *goja.Program, args *ExecuteArgs,
})
if err != nil {
if val, ok := err.(*goja.Exception); ok {
- err = val.Unwrap()
+ if x := val.Unwrap(); x != nil {
+ err = x
+ }
}
e := NewExecuteResult()
e["error"] = err.Error()
From 64885dcde67ee4b8fddcc8e89412e468719483b2 Mon Sep 17 00:00:00 2001
From: Dwi Siswanto
Date: Thu, 1 Aug 2024 16:28:22 +0700
Subject: [PATCH 120/283] docs(issue-report): add Nuclei command section
(#5477)
Signed-off-by: Dwi Siswanto
---
.github/ISSUE_TEMPLATE/issue-report.md | 5 +++++
1 file changed, 5 insertions(+)
diff --git a/.github/ISSUE_TEMPLATE/issue-report.md b/.github/ISSUE_TEMPLATE/issue-report.md
index 264b09cccc..3aa563b195 100644
--- a/.github/ISSUE_TEMPLATE/issue-report.md
+++ b/.github/ISSUE_TEMPLATE/issue-report.md
@@ -18,6 +18,11 @@ labels: 'Type: Bug'
+## Nuclei command:
+
+
+
+
### Current Behavior:
From 2655c29458367fa48e7a0e914a47f3a1d686ad32 Mon Sep 17 00:00:00 2001
From: Dogan Can Bakir <65292895+dogancanbakir@users.noreply.github.com>
Date: Thu, 1 Aug 2024 16:54:27 +0300
Subject: [PATCH 121/283] remove redundant code (#5479)
---
internal/runner/options.go | 8 +-------
1 file changed, 1 insertion(+), 7 deletions(-)
diff --git a/internal/runner/options.go b/internal/runner/options.go
index 8798c73113..2872b96a7a 100644
--- a/internal/runner/options.go
+++ b/internal/runner/options.go
@@ -19,7 +19,6 @@ import (
"github.com/projectdiscovery/gologger/levels"
"github.com/projectdiscovery/nuclei/v3/pkg/catalog/config"
"github.com/projectdiscovery/nuclei/v3/pkg/protocols/common/protocolinit"
- "github.com/projectdiscovery/nuclei/v3/pkg/protocols/common/protocolstate"
"github.com/projectdiscovery/nuclei/v3/pkg/protocols/common/utils/vardump"
"github.com/projectdiscovery/nuclei/v3/pkg/protocols/headless/engine"
"github.com/projectdiscovery/nuclei/v3/pkg/reporting"
@@ -115,12 +114,7 @@ func ParseOptions(options *types.Options) {
// Load the resolvers if user asked for them
loadResolvers(options)
- err := protocolstate.Init(options)
- if err != nil {
- gologger.Fatal().Msgf("Could not initialize protocol state: %s\n", err)
- }
-
- err = protocolinit.Init(options)
+ err := protocolinit.Init(options)
if err != nil {
gologger.Fatal().Msgf("Could not initialize protocols: %s\n", err)
}
From ff23949bb0f61ba9458faf3adc740233d7de3b8c Mon Sep 17 00:00:00 2001
From: Mohammed Diaa
Date: Thu, 1 Aug 2024 18:13:47 +0300
Subject: [PATCH 122/283] Apply input transformation to multi-protocol
templates (#5426)
* Apply input transformation to multi-protocol template execution
* Remove ad hoc input transoformation from DNS protocol
* Add SSL protocol input transformer
* Remove ad hoc input transoformation from SSL protocol
* Remove unused function extractDomain from the DNS protocol engine
* transform in flow as well
* bug fix + update test
* bug fix multi proto
:
* bug fix multi proto input
* bug fixes in input transform
---------
Co-authored-by: Tarun Koyalwar
---
pkg/input/transform.go | 13 ++++++++++-
pkg/input/transform_test.go | 4 ++--
pkg/protocols/common/contextargs/metainput.go | 1 +
pkg/protocols/dns/dns.go | 6 -----
pkg/protocols/dns/request.go | 23 ++-----------------
pkg/protocols/dns/request_test.go | 16 +------------
pkg/protocols/ssl/ssl.go | 19 +--------------
pkg/protocols/ssl/ssl_test.go | 5 ----
pkg/tmplexec/flow/flow_internal.go | 22 +++++++++++++++---
pkg/tmplexec/multiproto/multi.go | 14 +++++++----
pkg/tmplexec/multiproto/multi_test.go | 10 ++++++--
11 files changed, 55 insertions(+), 78 deletions(-)
diff --git a/pkg/input/transform.go b/pkg/input/transform.go
index 0afd649710..76f122c2e3 100644
--- a/pkg/input/transform.go
+++ b/pkg/input/transform.go
@@ -47,6 +47,8 @@ func (h *Helper) Transform(input string, protocol templateTypes.ProtocolType) st
return h.convertInputToType(input, typeHostWithOptionalPort, "")
case templateTypes.WebsocketProtocol:
return h.convertInputToType(input, typeWebsocket, "")
+ case templateTypes.SSLProtocol:
+ return h.convertInputToType(input, typeHostWithPort, "443")
}
return input
}
@@ -94,6 +96,8 @@ func (h *Helper) convertInputToType(input string, inputType inputType, defaultPo
if _, err := filepath.Match(input, ""); err != filepath.ErrBadPattern && !isURL {
return input
}
+ // if none of these satisfy the condition return empty
+ return ""
case typeHostOnly:
if hasHost {
return host
@@ -111,6 +115,10 @@ func (h *Helper) convertInputToType(input string, inputType inputType, defaultPo
return string(probed)
}
}
+ // try to parse it as absolute url and return
+ if absUrl, err := urlutil.ParseAbsoluteURL(input, false); err == nil {
+ return absUrl.String()
+ }
case typeHostWithPort, typeHostWithOptionalPort:
if hasHost && hasPort {
return net.JoinHostPort(host, port)
@@ -128,6 +136,9 @@ func (h *Helper) convertInputToType(input string, inputType inputType, defaultPo
if uri != nil && stringsutil.EqualFoldAny(uri.Scheme, "ws", "wss") {
return input
}
+ // empty if prefix is not given
+ return ""
}
- return ""
+ // do not return empty
+ return input
}
diff --git a/pkg/input/transform_test.go b/pkg/input/transform_test.go
index 76cabd3b53..699d877729 100644
--- a/pkg/input/transform_test.go
+++ b/pkg/input/transform_test.go
@@ -30,7 +30,7 @@ func TestConvertInputToType(t *testing.T) {
{"https://google.com:443", typeHostOnly, "google.com", ""},
// url
- {"test.com", typeURL, "", ""},
+ {"test.com", typeURL, "test.com", ""},
{"google.com", typeURL, "https://google.com", ""},
{"https://google.com", typeURL, "https://google.com", ""},
@@ -43,7 +43,7 @@ func TestConvertInputToType(t *testing.T) {
{"input_test.*", typeFilepath, "input_test.*", ""},
// host-port
- {"google.com", typeHostWithPort, "", ""},
+ {"google.com", typeHostWithPort, "google.com", ""},
{"google.com:443", typeHostWithPort, "google.com:443", ""},
{"https://google.com", typeHostWithPort, "google.com:443", ""},
{"https://google.com:443", typeHostWithPort, "google.com:443", ""},
diff --git a/pkg/protocols/common/contextargs/metainput.go b/pkg/protocols/common/contextargs/metainput.go
index b3c0dabe7a..d6515ed8e6 100644
--- a/pkg/protocols/common/contextargs/metainput.go
+++ b/pkg/protocols/common/contextargs/metainput.go
@@ -145,6 +145,7 @@ func (metaInput *MetaInput) Clone() *MetaInput {
input := NewMetaInput()
input.Input = metaInput.Input
input.CustomIP = metaInput.CustomIP
+ input.hash = metaInput.hash
if metaInput.ReqResp != nil {
input.ReqResp = metaInput.ReqResp.Clone()
}
diff --git a/pkg/protocols/dns/dns.go b/pkg/protocols/dns/dns.go
index 0a1bbca6cb..d6f462c440 100644
--- a/pkg/protocols/dns/dns.go
+++ b/pkg/protocols/dns/dns.go
@@ -141,12 +141,6 @@ func (request *Request) Compile(options *protocols.ExecutorOptions) error {
recursion := true
request.Recursion = &recursion
}
- dnsClientOptions := &dnsclientpool.Configuration{
- Retries: request.Retries,
- }
- if len(request.Resolvers) > 0 {
- dnsClientOptions.Resolvers = request.Resolvers
- }
// Create a dns client for the class
client, err := request.getDnsClient(options, nil)
if err != nil {
diff --git a/pkg/protocols/dns/request.go b/pkg/protocols/dns/request.go
index b4f04118e5..83501c15b1 100644
--- a/pkg/protocols/dns/request.go
+++ b/pkg/protocols/dns/request.go
@@ -3,7 +3,6 @@ package dns
import (
"encoding/hex"
"fmt"
- "net/url"
"strings"
"sync"
@@ -23,7 +22,6 @@ import (
"github.com/projectdiscovery/nuclei/v3/pkg/protocols/common/utils/vardump"
protocolutils "github.com/projectdiscovery/nuclei/v3/pkg/protocols/utils"
templateTypes "github.com/projectdiscovery/nuclei/v3/pkg/templates/types"
- "github.com/projectdiscovery/nuclei/v3/pkg/utils"
"github.com/projectdiscovery/retryabledns"
iputil "github.com/projectdiscovery/utils/ip"
syncutil "github.com/projectdiscovery/utils/sync"
@@ -38,16 +36,8 @@ func (request *Request) Type() templateTypes.ProtocolType {
// ExecuteWithResults executes the protocol requests and returns results instead of writing them.
func (request *Request) ExecuteWithResults(input *contextargs.Context, metadata, previous output.InternalEvent, callback protocols.OutputEventCallback) error {
- // Parse the URL and return domain if URL.
- var domain string
- if utils.IsURL(input.MetaInput.Input) {
- domain = extractDomain(input.MetaInput.Input)
- } else {
- domain = input.MetaInput.Input
- }
-
var err error
- domain, err = request.parseDNSInput(domain)
+ domain, err := request.parseDNSInput(input.MetaInput.Input)
if err != nil {
return errors.Wrap(err, "could not build request")
}
@@ -230,7 +220,7 @@ func (request *Request) parseDNSInput(host string) (string, error) {
return host, nil
}
-func dumpResponse(event *output.InternalWrappedEvent, request *Request, requestOptions *protocols.ExecutorOptions, response, domain string) {
+func dumpResponse(event *output.InternalWrappedEvent, request *Request, _ *protocols.ExecutorOptions, response, domain string) {
cliOptions := request.options.Options
if cliOptions.Debug || cliOptions.DebugResponse || cliOptions.StoreResponse {
hexDump := false
@@ -261,12 +251,3 @@ func dumpTraceData(event *output.InternalWrappedEvent, requestOptions *protocols
gologger.Debug().Msgf("[%s] Dumped DNS Trace data for %s\n\n%s", requestOptions.TemplateID, domain, highlightedResponse)
}
}
-
-// extractDomain extracts the domain name of a URL
-func extractDomain(theURL string) string {
- u, err := url.Parse(theURL)
- if err != nil {
- return ""
- }
- return u.Hostname()
-}
diff --git a/pkg/protocols/dns/request_test.go b/pkg/protocols/dns/request_test.go
index 81f0b98ab2..6410de7080 100644
--- a/pkg/protocols/dns/request_test.go
+++ b/pkg/protocols/dns/request_test.go
@@ -67,19 +67,5 @@ func TestDNSExecuteWithResults(t *testing.T) {
require.Equal(t, 1, len(finalEvent.Results[0].ExtractedResults), "could not get correct number of extracted results")
require.Equal(t, "93.184.215.14", finalEvent.Results[0].ExtractedResults[0], "could not get correct extracted results")
finalEvent = nil
-
- t.Run("url-to-domain", func(t *testing.T) {
- metadata := make(output.InternalEvent)
- previous := make(output.InternalEvent)
- err := request.ExecuteWithResults(contextargs.NewWithInput(context.Background(), "https://example.com"), metadata, previous, func(event *output.InternalWrappedEvent) {
- finalEvent = event
- })
- require.Nil(t, err, "could not execute dns request")
- })
- require.NotNil(t, finalEvent, "could not get event output from request")
- require.Equal(t, 1, len(finalEvent.Results), "could not get correct number of results")
- require.Equal(t, "test", finalEvent.Results[0].MatcherName, "could not get correct matcher name of results")
- require.Equal(t, 1, len(finalEvent.Results[0].ExtractedResults), "could not get correct number of extracted results")
- require.Equal(t, "93.184.215.14", finalEvent.Results[0].ExtractedResults[0], "could not get correct extracted results")
- finalEvent = nil
+ // Note: changing url to domain is responsible at tmplexec package and is implemented there
}
diff --git a/pkg/protocols/ssl/ssl.go b/pkg/protocols/ssl/ssl.go
index 1abde973aa..50da53111b 100644
--- a/pkg/protocols/ssl/ssl.go
+++ b/pkg/protocols/ssl/ssl.go
@@ -34,7 +34,6 @@ import (
"github.com/projectdiscovery/tlsx/pkg/tlsx/openssl"
errorutil "github.com/projectdiscovery/utils/errors"
stringsutil "github.com/projectdiscovery/utils/strings"
- urlutil "github.com/projectdiscovery/utils/url"
)
// Request is a request for the SSL protocol
@@ -199,10 +198,7 @@ func (request *Request) GetID() string {
// ExecuteWithResults executes the protocol requests and returns results instead of writing them.
func (request *Request) ExecuteWithResults(input *contextargs.Context, dynamicValues, previous output.InternalEvent, callback protocols.OutputEventCallback) error {
- hostPort, err := getAddress(input.MetaInput.Input)
- if err != nil {
- return err
- }
+ hostPort := input.MetaInput.Input
hostname, port, _ := net.SplitHostPort(hostPort)
requestOptions := request.options
@@ -358,19 +354,6 @@ var RequestPartDefinitions = map[string]string{
"matched": "Matched is the input which was matched upon",
}
-// getAddress returns the address of the host to make request to
-func getAddress(toTest string) (string, error) {
- urlx, err := urlutil.Parse(toTest)
- if err != nil {
- // use given input instead of url parsing failure
- return toTest, nil
- }
- if urlx.Port() == "" {
- urlx.UpdatePort("443")
- }
- return urlx.Host, nil
-}
-
// Match performs matching operation for a matcher on model and returns:
// true and a list of matched snippets if the matcher type is supports it
// otherwise false and an empty string slice
diff --git a/pkg/protocols/ssl/ssl_test.go b/pkg/protocols/ssl/ssl_test.go
index 59c9f85f3e..52a3ae5077 100644
--- a/pkg/protocols/ssl/ssl_test.go
+++ b/pkg/protocols/ssl/ssl_test.go
@@ -36,8 +36,3 @@ func TestSSLProtocol(t *testing.T) {
require.Nil(t, err, "could not run ssl request")
require.NotEmpty(t, gotEvent, "could not get event items")
}
-
-func TestGetAddress(t *testing.T) {
- address, _ := getAddress("https://scanme.sh")
- require.Equal(t, "scanme.sh:443", address, "could not get correct address")
-}
diff --git a/pkg/tmplexec/flow/flow_internal.go b/pkg/tmplexec/flow/flow_internal.go
index 6e82bd960e..e63153c547 100644
--- a/pkg/tmplexec/flow/flow_internal.go
+++ b/pkg/tmplexec/flow/flow_internal.go
@@ -29,7 +29,15 @@ func (f *FlowExecutor) requestExecutor(runtime *goja.Runtime, reqMap mapsutil.Ma
// execution logic for http()/dns() etc
for index := range f.allProtocols[opts.protoName] {
req := f.allProtocols[opts.protoName][index]
- err := req.ExecuteWithResults(f.ctx.Input, output.InternalEvent(f.options.GetTemplateCtx(f.ctx.Input.MetaInput).GetAll()), nil, f.protocolResultCallback(req, matcherStatus, opts))
+ // transform input if required
+ inputItem := f.ctx.Input.Clone()
+ if f.options.InputHelper != nil && f.ctx.Input.MetaInput.Input != "" {
+ if inputItem.MetaInput.Input = f.options.InputHelper.Transform(inputItem.MetaInput.Input, req.Type()); inputItem.MetaInput.Input == "" {
+ f.ctx.LogError(fmt.Errorf("failed to transform input for protocol %s", req.Type()))
+ return false
+ }
+ }
+ err := req.ExecuteWithResults(inputItem, output.InternalEvent(f.options.GetTemplateCtx(f.ctx.Input.MetaInput).GetAll()), nil, f.protocolResultCallback(req, matcherStatus, opts))
if err != nil {
// save all errors in a map with id as key
// its less likely that there will be race condition but just in case
@@ -58,7 +66,15 @@ func (f *FlowExecutor) requestExecutor(runtime *goja.Runtime, reqMap mapsutil.Ma
}
return matcherStatus.Load()
}
- err := req.ExecuteWithResults(f.ctx.Input, output.InternalEvent(f.options.GetTemplateCtx(f.ctx.Input.MetaInput).GetAll()), nil, f.protocolResultCallback(req, matcherStatus, opts))
+ // transform input if required
+ inputItem := f.ctx.Input.Clone()
+ if f.options.InputHelper != nil && f.ctx.Input.MetaInput.Input != "" {
+ if inputItem.MetaInput.Input = f.options.InputHelper.Transform(inputItem.MetaInput.Input, req.Type()); inputItem.MetaInput.Input == "" {
+ f.ctx.LogError(fmt.Errorf("failed to transform input for protocol %s", req.Type()))
+ return false
+ }
+ }
+ err := req.ExecuteWithResults(inputItem, output.InternalEvent(f.options.GetTemplateCtx(f.ctx.Input.MetaInput).GetAll()), nil, f.protocolResultCallback(req, matcherStatus, opts))
if err != nil {
index := id
err = f.allErrs.Set(opts.protoName+":"+index, err)
@@ -72,7 +88,7 @@ func (f *FlowExecutor) requestExecutor(runtime *goja.Runtime, reqMap mapsutil.Ma
// protocolResultCallback returns a callback that is executed
// after execution of each protocol request
-func (f *FlowExecutor) protocolResultCallback(req protocols.Request, matcherStatus *atomic.Bool, opts *ProtoOptions) func(result *output.InternalWrappedEvent) {
+func (f *FlowExecutor) protocolResultCallback(req protocols.Request, matcherStatus *atomic.Bool, _ *ProtoOptions) func(result *output.InternalWrappedEvent) {
return func(result *output.InternalWrappedEvent) {
if result != nil {
// Note: flow specific implicit behaviours should be handled here
diff --git a/pkg/tmplexec/multiproto/multi.go b/pkg/tmplexec/multiproto/multi.go
index 7bbc2a1403..87fc35c00a 100644
--- a/pkg/tmplexec/multiproto/multi.go
+++ b/pkg/tmplexec/multiproto/multi.go
@@ -109,14 +109,19 @@ func (m *MultiProtocol) ExecuteWithResults(ctx *scan.ScanContext) error {
return ctx.Context().Err()
default:
}
-
- values := m.options.GetTemplateCtx(ctx.Input.MetaInput).GetAll()
- err := req.ExecuteWithResults(ctx.Input, output.InternalEvent(values), nil, multiProtoCallback)
+ inputItem := ctx.Input.Clone()
+ if m.options.InputHelper != nil && ctx.Input.MetaInput.Input != "" {
+ if inputItem.MetaInput.Input = m.options.InputHelper.Transform(inputItem.MetaInput.Input, req.Type()); inputItem.MetaInput.Input == "" {
+ return nil
+ }
+ }
+ // FIXME: this hack of using hash to get templateCtx has known issues scan context based approach should be adopted ASAP
+ values := m.options.GetTemplateCtx(inputItem.MetaInput).GetAll()
+ err := req.ExecuteWithResults(inputItem, output.InternalEvent(values), nil, multiProtoCallback)
// in case of fatal error skip execution of next protocols
if err != nil {
// always log errors
ctx.LogError(err)
-
// for some classes of protocols (i.e ssl) errors like tls handshake are a legitimate behavior so we don't stop execution
// connection failures are already tracked by the internal host error cache
// we use strings comparison as the error is not formalized into instance within the standard library
@@ -124,7 +129,6 @@ func (m *MultiProtocol) ExecuteWithResults(ctx *scan.ScanContext) error {
if req.Type() == types.SSLProtocol && stringsutil.ContainsAnyI(err.Error(), "protocol version not supported", "could not do tls handshake") {
continue
}
-
return err
}
}
diff --git a/pkg/tmplexec/multiproto/multi_test.go b/pkg/tmplexec/multiproto/multi_test.go
index 4f2aa25e4e..2750b8cd64 100644
--- a/pkg/tmplexec/multiproto/multi_test.go
+++ b/pkg/tmplexec/multiproto/multi_test.go
@@ -3,11 +3,13 @@ package multiproto_test
import (
"context"
"log"
+ "os"
"testing"
"time"
"github.com/projectdiscovery/nuclei/v3/pkg/catalog/config"
"github.com/projectdiscovery/nuclei/v3/pkg/catalog/disk"
+ "github.com/projectdiscovery/nuclei/v3/pkg/input"
"github.com/projectdiscovery/nuclei/v3/pkg/loader/workflow"
"github.com/projectdiscovery/nuclei/v3/pkg/progress"
"github.com/projectdiscovery/nuclei/v3/pkg/protocols"
@@ -36,6 +38,7 @@ func setup() {
Catalog: disk.NewCatalog(config.DefaultConfig.TemplatesDirectory),
RateLimiter: ratelimit.New(context.Background(), uint(options.RateLimit), time.Second),
Parser: templates.NewParser(),
+ InputHelper: input.NewHelper(),
}
workflowLoader, err := workflow.NewLoader(&executerOpts)
if err != nil {
@@ -45,7 +48,6 @@ func setup() {
}
func TestMultiProtoWithDynamicExtractor(t *testing.T) {
- setup()
Template, err := templates.Parse("testcases/multiprotodynamic.yaml", nil, executerOpts)
require.Nil(t, err, "could not parse template")
@@ -62,7 +64,6 @@ func TestMultiProtoWithDynamicExtractor(t *testing.T) {
}
func TestMultiProtoWithProtoPrefix(t *testing.T) {
- setup()
Template, err := templates.Parse("testcases/multiprotowithprefix.yaml", nil, executerOpts)
require.Nil(t, err, "could not parse template")
@@ -77,3 +78,8 @@ func TestMultiProtoWithProtoPrefix(t *testing.T) {
require.Nil(t, err, "could not execute template")
require.True(t, gotresults)
}
+
+func TestMain(m *testing.M) {
+ setup()
+ os.Exit(m.Run())
+}
From 0787ff29e0e1380e89fa73ba298b5f12a9afa028 Mon Sep 17 00:00:00 2001
From: alban-stourbe-wmx <159776828+alban-stourbe-wmx@users.noreply.github.com>
Date: Sat, 3 Aug 2024 19:43:31 +0200
Subject: [PATCH 123/283] Add Workflows SDK scan (#5409)
* Add Workflows SDK scan
* minor
---------
Co-authored-by: Mzack9999
---
lib/sdk.go | 7 ++++++-
1 file changed, 6 insertions(+), 1 deletion(-)
diff --git a/lib/sdk.go b/lib/sdk.go
index 60f5255bcd..04b084f244 100644
--- a/lib/sdk.go
+++ b/lib/sdk.go
@@ -243,7 +243,12 @@ func (e *NucleiEngine) ExecuteCallbackWithCtx(ctx context.Context, callback ...f
}
e.resultCallbacks = append(e.resultCallbacks, filtered...)
- _ = e.engine.ExecuteScanWithOpts(ctx, e.store.Templates(), e.inputProvider, false)
+ templatesAndWorkflows := append(e.store.Templates(), e.store.Workflows()...)
+ if len(templatesAndWorkflows) == 0 {
+ return ErrNoTemplatesAvailable
+ }
+
+ _ = e.engine.ExecuteScanWithOpts(ctx, templatesAndWorkflows, e.inputProvider, false)
defer e.engine.WorkPool().Wait()
return nil
}
From ddcc9211beb18dea51710728386b77e078f80ece Mon Sep 17 00:00:00 2001
From: Dwi Siswanto
Date: Sun, 4 Aug 2024 01:14:37 +0700
Subject: [PATCH 124/283] ci: use composite actions (#5483)
Signed-off-by: Dwi Siswanto
---
.github/workflows/build-test.yml | 15 ++++++---------
.github/workflows/functional-test.yml | 10 ++++------
.github/workflows/lint-test.yml | 15 +++++----------
.github/workflows/performance-test.yaml | 11 ++++-------
.github/workflows/publish-docs.yaml | 11 +++++------
.github/workflows/release-binary.yml | 13 +++++--------
.github/workflows/release-test.yml | 13 ++++---------
.github/workflows/template-validate.yml | 7 +++----
8 files changed, 36 insertions(+), 59 deletions(-)
diff --git a/.github/workflows/build-test.yml b/.github/workflows/build-test.yml
index 75591aa8b8..8ee149c79d 100644
--- a/.github/workflows/build-test.yml
+++ b/.github/workflows/build-test.yml
@@ -12,24 +12,21 @@ jobs:
name: Test Builds
strategy:
matrix:
- go-version: [1.21.x]
os: [ubuntu-latest, windows-latest, macOS-latest]
-
runs-on: ${{ matrix.os }}
steps:
+ - name: Check out code
+ uses: actions/checkout@v4
+
- name: Set up Go
- uses: actions/setup-go@v4
- with:
- go-version: ${{ matrix.go-version }}
+ uses: projectdiscovery/actions/setup/go@v1
- - name: Set up Python # required for running python code in py-snippet.yaml integration test
+ # required for running python code in py-snippet.yaml integration test
+ - name: Set up Python
uses: actions/setup-python@v4
with:
python-version: '3.10'
- - name: Check out code
- uses: actions/checkout@v3
-
- name: Go Mod hygiene
run: |
go clean -modcache
diff --git a/.github/workflows/functional-test.yml b/.github/workflows/functional-test.yml
index 912080d8a0..cfe8ab1465 100644
--- a/.github/workflows/functional-test.yml
+++ b/.github/workflows/functional-test.yml
@@ -15,13 +15,11 @@ jobs:
matrix:
os: [ubuntu-latest, windows-latest, macOS-latest]
steps:
- - name: Set up Go
- uses: actions/setup-go@v4
- with:
- go-version: 1.21.x
-
- name: Check out code
- uses: actions/checkout@v3
+ uses: actions/checkout@v4
+
+ - name: Set up Go
+ uses: projectdiscovery/actions/setup/go@v1
- name: Functional Tests
env:
diff --git a/.github/workflows/lint-test.yml b/.github/workflows/lint-test.yml
index 8b4695dc83..cd8ceed9ca 100644
--- a/.github/workflows/lint-test.yml
+++ b/.github/workflows/lint-test.yml
@@ -12,16 +12,11 @@ jobs:
name: Lint Test
runs-on: ubuntu-latest
steps:
- - name: Set up Go
- uses: actions/setup-go@v4
- with:
- go-version: 1.21.x
-
- name: Checkout code
- uses: actions/checkout@v3
+ uses: actions/checkout@v4
+
+ - name: Set up Go
+ uses: projectdiscovery/actions/setup/go@v1
- name: Run golangci-lint
- uses: golangci/golangci-lint-action@v4.0.0
- with:
- version: latest
- args: --timeout 5m
\ No newline at end of file
+ uses: projectdiscovery/actions/golangci-lint@v1
diff --git a/.github/workflows/performance-test.yaml b/.github/workflows/performance-test.yaml
index 76c24bf916..6baecfaff2 100644
--- a/.github/workflows/performance-test.yaml
+++ b/.github/workflows/performance-test.yaml
@@ -11,19 +11,16 @@ jobs:
name: Test Performance
strategy:
matrix:
- go-version: [1.21.x]
os: [ubuntu-latest, macOS-latest]
runs-on: ${{ matrix.os }}
if: github.repository == 'projectdiscovery/nuclei'
steps:
- - name: Set up Go
- uses: actions/setup-go@v4
- with:
- go-version: ${{ matrix.go-version }}
-
- name: Check out code
- uses: actions/checkout@v3
+ uses: actions/checkout@v4
+
+ - name: Set up Go
+ uses: projectdiscovery/actions/setup/go@v1
- name: Go Mod hygine
run: |
diff --git a/.github/workflows/publish-docs.yaml b/.github/workflows/publish-docs.yaml
index da3a5110cd..72adb4b0ad 100644
--- a/.github/workflows/publish-docs.yaml
+++ b/.github/workflows/publish-docs.yaml
@@ -11,14 +11,15 @@ jobs:
runs-on: ubuntu-latest
steps:
- name: Checkout code
- uses: actions/checkout@v3
+ uses: actions/checkout@v4
with:
fetch-depth: 0
- name: "Set up Go"
- uses: actions/setup-go@v4
- with:
- go-version: 1.21.x
+ uses: projectdiscovery/actions/setup/go@v1
+
+ - name: "Set up Git"
+ uses: projectdiscovery/actions/setup/git@v1
- name: Generate YAML Syntax Documentation
id: generate-docs
@@ -35,8 +36,6 @@ jobs:
- name: Commit files
if: steps.generate-docs.outputs.CHANGES > 0
run: |
- git config --local user.email "action@github.com"
- git config --local user.name "GitHub Action"
git add SYNTAX-REFERENCE.md nuclei-jsonschema.json
git commit -m "Auto Generate Syntax Docs + JSONSchema [$(date)] :robot:" -a
diff --git a/.github/workflows/release-binary.yml b/.github/workflows/release-binary.yml
index 5c7181776a..b4827c8c2d 100644
--- a/.github/workflows/release-binary.yml
+++ b/.github/workflows/release-binary.yml
@@ -9,20 +9,17 @@ on:
jobs:
release:
runs-on: ubuntu-latest-16-cores
-
steps:
- - uses: actions/checkout@v3
+ - uses: actions/checkout@v4
with:
fetch-depth: 0
- - uses: actions/setup-go@v4
- with:
- go-version: 1.21.x
+ - name: Set up Go
+ uses: projectdiscovery/actions/setup/go@v1
- - uses: goreleaser/goreleaser-action@v4
+ - uses: projectdiscovery/actions/goreleaser@v1
with:
- args: "release --clean"
- version: latest
+ release: true
env:
GITHUB_TOKEN: "${{ secrets.GITHUB_TOKEN }}"
SLACK_WEBHOOK: "${{ secrets.RELEASE_SLACK_WEBHOOK }}"
diff --git a/.github/workflows/release-test.yml b/.github/workflows/release-test.yml
index 4a13baae63..b4edbc728b 100644
--- a/.github/workflows/release-test.yml
+++ b/.github/workflows/release-test.yml
@@ -12,17 +12,12 @@ jobs:
runs-on: ubuntu-latest-16-cores
steps:
- name: "Check out code"
- uses: actions/checkout@v3
+ uses: actions/checkout@v4
with:
fetch-depth: 0
- name: Set up Go
- uses: actions/setup-go@v4
- with:
- go-version: 1.21.x
+ uses: projectdiscovery/actions/setup/go@v1
- - name: release test
- uses: goreleaser/goreleaser-action@v4
- with:
- args: "release --clean --snapshot"
- version: latest
\ No newline at end of file
+ - name: Release snapshot
+ uses: projectdiscovery/actions/goreleaser@v1
diff --git a/.github/workflows/template-validate.yml b/.github/workflows/template-validate.yml
index a3ca5bd9f1..05279def57 100644
--- a/.github/workflows/template-validate.yml
+++ b/.github/workflows/template-validate.yml
@@ -10,10 +10,9 @@ jobs:
build:
runs-on: ubuntu-latest-16-cores
steps:
- - uses: actions/checkout@v3
- - uses: actions/setup-go@v4
- with:
- go-version: 1.21.x
+ - uses: actions/checkout@v4
+
+ - uses: projectdiscovery/actions/setup/go@v1
- name: Template Validation
run: |
From 2df1b2e88e464b76b32e98f260dd6091fe429edf Mon Sep 17 00:00:00 2001
From: Tarun Koyalwar <45962551+tarunKoyalwar@users.noreply.github.com>
Date: Sun, 4 Aug 2024 18:14:08 +0530
Subject: [PATCH 125/283] file proto missing vars in flow & multi-protocol
(#5480)
* fix missing template context in file proto
* fix file protocol missing vars
* fix test
* skip example advanced test
---
.github/workflows/build-test.yml | 7 ++++---
pkg/protocols/file/request.go | 6 +++++-
pkg/protocols/protocols.go | 10 ++++++++++
pkg/templates/types/types.go | 12 ++++++++++++
pkg/tmplexec/flow/flow_executor_test.go | 19 ++++++++++---------
5 files changed, 41 insertions(+), 13 deletions(-)
diff --git a/.github/workflows/build-test.yml b/.github/workflows/build-test.yml
index 8ee149c79d..92e2b479cd 100644
--- a/.github/workflows/build-test.yml
+++ b/.github/workflows/build-test.yml
@@ -62,9 +62,10 @@ jobs:
run: go run .
working-directory: examples/simple/
- - name: Example SDK Advanced
- run: go run .
- working-directory: examples/advanced/
+ # Temporarily disabled very flaky in github actions
+ # - name: Example SDK Advanced
+ # run: go run .
+ # working-directory: examples/advanced/
- name: Example SDK with speed control
run: go run .
diff --git a/pkg/protocols/file/request.go b/pkg/protocols/file/request.go
index f13f08d1ab..b7d8d086b7 100644
--- a/pkg/protocols/file/request.go
+++ b/pkg/protocols/file/request.go
@@ -51,6 +51,9 @@ func (request *Request) ExecuteWithResults(input *contextargs.Context, metadata,
if err != nil {
return err
}
+ if input.MetaInput.Input == "" {
+ return errors.New("input cannot be empty file or folder expected")
+ }
err = request.getInputPaths(input.MetaInput.Input, func(filePath string) {
wg.Add()
func(filePath string) {
@@ -250,6 +253,8 @@ func (request *Request) findMatchesWithReader(reader io.Reader, input *contextar
for k, v := range previous {
dslMap[k] = v
}
+ // add vars to template context
+ request.options.AddTemplateVars(input.MetaInput, request.Type(), request.ID, dslMap)
// add template context variables to DSL map
if request.options.HasTemplateCtx(input.MetaInput) {
dslMap = generators.MergeMaps(dslMap, request.options.GetTemplateCtx(input.MetaInput).GetAll())
@@ -323,7 +328,6 @@ func (request *Request) buildEvent(input, filePath string, fileMatches []FileMat
exprLines[fileMatch.Expr] = append(exprLines[fileMatch.Expr], fileMatch.Line)
exprBytes[fileMatch.Expr] = append(exprBytes[fileMatch.Expr], fileMatch.ByteIndex)
}
-
event := eventcreator.CreateEventWithOperatorResults(request, internalEvent, operatorResult)
// Annotate with line numbers if asked by the user
if request.options.Options.ShowMatchLine {
diff --git a/pkg/protocols/protocols.go b/pkg/protocols/protocols.go
index af7d2b4766..8b8854cd8b 100644
--- a/pkg/protocols/protocols.go
+++ b/pkg/protocols/protocols.go
@@ -197,6 +197,11 @@ func (e *ExecutorOptions) AddTemplateVars(input *contextargs.MetaInput, reqType
}
templateCtx := e.GetTemplateCtx(input)
for k, v := range vars {
+ if stringsutil.HasPrefixAny(k, templateTypes.SupportedProtocolsStrings()...) {
+ // this was inherited from previous protocols no need to modify it we can directly set it or omit
+ templateCtx.Set(k, v)
+ continue
+ }
if !stringsutil.EqualFoldAny(k, "template-id", "template-info", "template-path") {
if reqID != "" {
k = reqID + "_" + k
@@ -216,6 +221,11 @@ func (e *ExecutorOptions) AddTemplateVar(input *contextargs.MetaInput, templateT
return
}
templateCtx := e.GetTemplateCtx(input)
+ if stringsutil.HasPrefixAny(key, templateTypes.SupportedProtocolsStrings()...) {
+ // this was inherited from previous protocols no need to modify it we can directly set it or omit
+ templateCtx.Set(key, value)
+ return
+ }
if reqID != "" {
key = reqID + "_" + key
} else if templateType < templateTypes.InvalidProtocol {
diff --git a/pkg/templates/types/types.go b/pkg/templates/types/types.go
index f51c444f8b..164deb68b3 100644
--- a/pkg/templates/types/types.go
+++ b/pkg/templates/types/types.go
@@ -69,6 +69,18 @@ func GetSupportedProtocolTypes() ProtocolTypes {
return result
}
+// SupportedProtocolsStrings returns a slice of strings of supported protocols
+func SupportedProtocolsStrings() []string {
+ var result []string
+ for _, protocol := range GetSupportedProtocolTypes() {
+ if protocol.String() == "" {
+ continue
+ }
+ result = append(result, protocol.String())
+ }
+ return result
+}
+
func toProtocolType(valueToMap string) (ProtocolType, error) {
normalizedValue := normalizeValue(valueToMap)
for key, currentValue := range protocolMappings {
diff --git a/pkg/tmplexec/flow/flow_executor_test.go b/pkg/tmplexec/flow/flow_executor_test.go
index cf7b1790a6..fd4da0d6f7 100644
--- a/pkg/tmplexec/flow/flow_executor_test.go
+++ b/pkg/tmplexec/flow/flow_executor_test.go
@@ -27,15 +27,15 @@ func setup() {
progressImpl, _ := progress.NewStatsTicker(0, false, false, false, 0)
executerOpts = protocols.ExecutorOptions{
- Output: testutils.NewMockOutputWriter(options.OmitTemplate),
- Options: options,
- Progress: progressImpl,
- ProjectFile: nil,
- IssuesClient: nil,
- Browser: nil,
- Catalog: disk.NewCatalog(config.DefaultConfig.TemplatesDirectory),
- RateLimiter: ratelimit.New(context.Background(), uint(options.RateLimit), time.Second),
- Parser: templates.NewParser(),
+ Output: testutils.NewMockOutputWriter(options.OmitTemplate),
+ Options: options,
+ Progress: progressImpl,
+ ProjectFile: nil,
+ IssuesClient: nil,
+ Browser: nil,
+ Catalog: disk.NewCatalog(config.DefaultConfig.TemplatesDirectory),
+ RateLimiter: ratelimit.New(context.Background(), uint(options.RateLimit), time.Second),
+ Parser: templates.NewParser(),
}
workflowLoader, err := workflow.NewLoader(&executerOpts)
if err != nil {
@@ -146,6 +146,7 @@ func TestFlowWithConditionPositive(t *testing.T) {
}
func TestFlowWithNoMatchers(t *testing.T) {
+ setup()
// when using conditional flow with no matchers at all
// we implicitly assume that request was successful and internally changed the result to true (for scope of condition only)
From 471d6730f477f8d7af8149dbb169610ddc6cae0b Mon Sep 17 00:00:00 2001
From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com>
Date: Mon, 5 Aug 2024 05:45:47 +0000
Subject: [PATCH 126/283] chore(deps): bump
github.com/projectdiscovery/wappalyzergo
Bumps [github.com/projectdiscovery/wappalyzergo](https://github.com/projectdiscovery/wappalyzergo) from 0.1.11 to 0.1.13.
- [Release notes](https://github.com/projectdiscovery/wappalyzergo/releases)
- [Commits](https://github.com/projectdiscovery/wappalyzergo/compare/v0.1.11...v0.1.13)
---
updated-dependencies:
- dependency-name: github.com/projectdiscovery/wappalyzergo
dependency-type: direct:production
update-type: version-update:semver-patch
...
Signed-off-by: dependabot[bot]
---
go.mod | 2 +-
go.sum | 4 ++--
2 files changed, 3 insertions(+), 3 deletions(-)
diff --git a/go.mod b/go.mod
index d9c722509f..1d1e96ed93 100644
--- a/go.mod
+++ b/go.mod
@@ -96,7 +96,7 @@ require (
github.com/projectdiscovery/uncover v1.0.9
github.com/projectdiscovery/useragent v0.0.60
github.com/projectdiscovery/utils v0.2.3
- github.com/projectdiscovery/wappalyzergo v0.1.11
+ github.com/projectdiscovery/wappalyzergo v0.1.13
github.com/redis/go-redis/v9 v9.1.0
github.com/seh-msft/burpxml v1.0.1
github.com/stretchr/testify v1.9.0
diff --git a/go.sum b/go.sum
index d9973196fa..d43e561a85 100644
--- a/go.sum
+++ b/go.sum
@@ -888,8 +888,8 @@ github.com/projectdiscovery/useragent v0.0.60 h1:qDU1rwA+XOKmSqp7yoijAN4PuvLQc2Z
github.com/projectdiscovery/useragent v0.0.60/go.mod h1:05IDiJEy2dWl3x6dnsWtJYPwT40oWha144Us7+Fwr6w=
github.com/projectdiscovery/utils v0.2.3 h1:rkambl0EoTF/y6DpjCfSwcVUFdkAeVOtYkK3lX6InCY=
github.com/projectdiscovery/utils v0.2.3/go.mod h1:eGuuQ5Acekg47WsFS1Q9Qxw8+vI6IxwqIQSAplBBG0c=
-github.com/projectdiscovery/wappalyzergo v0.1.11 h1:6RZFcu4XcZ7nxi0iSI7hfq/Ivb/eEIF8h7fnajyoibo=
-github.com/projectdiscovery/wappalyzergo v0.1.11/go.mod h1:/hzgxkBFTMe2wDbA93nFfoMjULw7/vIZ9QPSAnCgUa8=
+github.com/projectdiscovery/wappalyzergo v0.1.13 h1:tQ/pjdW4w2NHqUiFlNe/dULrPH8A9GOfnmplDzEOp3M=
+github.com/projectdiscovery/wappalyzergo v0.1.13/go.mod h1:/hzgxkBFTMe2wDbA93nFfoMjULw7/vIZ9QPSAnCgUa8=
github.com/projectdiscovery/yamldoc-go v1.0.4 h1:eZoESapnMw6WAHiVgRwNqvbJEfNHEH148uthhFbG5jE=
github.com/projectdiscovery/yamldoc-go v1.0.4/go.mod h1:8PIPRcUD55UbtQdcfFR1hpIGRWG0P7alClXNGt1TBik=
github.com/prometheus/client_golang v0.9.1/go.mod h1:7SWBe2y4D6OKWSNQJUaRYU/AaXPKyh/dDVn+NZz0KFw=
From 6179e9dd4172b16fe7e79a5e8d2eedf94231f1c4 Mon Sep 17 00:00:00 2001
From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com>
Date: Mon, 5 Aug 2024 05:46:02 +0000
Subject: [PATCH 127/283] chore(deps): bump
github.com/projectdiscovery/gologger
Bumps [github.com/projectdiscovery/gologger](https://github.com/projectdiscovery/gologger) from 1.1.16 to 1.1.18.
- [Release notes](https://github.com/projectdiscovery/gologger/releases)
- [Commits](https://github.com/projectdiscovery/gologger/compare/v1.1.16...v1.1.18)
---
updated-dependencies:
- dependency-name: github.com/projectdiscovery/gologger
dependency-type: direct:production
update-type: version-update:semver-patch
...
Signed-off-by: dependabot[bot]
---
go.mod | 2 +-
go.sum | 4 ++--
2 files changed, 3 insertions(+), 3 deletions(-)
diff --git a/go.mod b/go.mod
index d9c722509f..73b226ef8d 100644
--- a/go.mod
+++ b/go.mod
@@ -83,7 +83,7 @@ require (
github.com/projectdiscovery/fasttemplate v0.0.2
github.com/projectdiscovery/go-smb2 v0.0.0-20240129202741-052cc450c6cb
github.com/projectdiscovery/goflags v0.1.59
- github.com/projectdiscovery/gologger v1.1.16
+ github.com/projectdiscovery/gologger v1.1.18
github.com/projectdiscovery/gostruct v0.0.2
github.com/projectdiscovery/gozero v0.0.2
github.com/projectdiscovery/httpx v1.6.6
diff --git a/go.sum b/go.sum
index d9973196fa..a525d15be5 100644
--- a/go.sum
+++ b/go.sum
@@ -844,8 +844,8 @@ github.com/projectdiscovery/go-smb2 v0.0.0-20240129202741-052cc450c6cb h1:rutG90
github.com/projectdiscovery/go-smb2 v0.0.0-20240129202741-052cc450c6cb/go.mod h1:FLjF1DmZ+POoGEiIQdWuYVwS++C/GwpX8YaCsTSm1RY=
github.com/projectdiscovery/goflags v0.1.59 h1:rFScpC57L0Ln0dJmBtstyxHUH7K9H4UmHACXcSkdqqk=
github.com/projectdiscovery/goflags v0.1.59/go.mod h1:dj67QGp/D81WRLtzU0HsWR20zgoGZ0cnk3Wbt9xJcuo=
-github.com/projectdiscovery/gologger v1.1.16 h1:NsYQVNt1H8O3Wmag4sXxQxvbLbIP16bjCm/cnCvf5hc=
-github.com/projectdiscovery/gologger v1.1.16/go.mod h1:WlyfroigIqU/in8A3fTEeMJ6t5NfbCG+rgWcvI5dQiQ=
+github.com/projectdiscovery/gologger v1.1.18 h1:fux2S/LXYe18/w4PylAFpEFg+EApNCuyS85x25nDE98=
+github.com/projectdiscovery/gologger v1.1.18/go.mod h1:IgjKgVLfVGUovxlC6WJcXK0qvo/tNZ1P0+r6miQqyRk=
github.com/projectdiscovery/gostruct v0.0.2 h1:s8gP8ApugGM4go1pA+sVlPDXaWqNP5BBDDSv7VEdG1M=
github.com/projectdiscovery/gostruct v0.0.2/go.mod h1:H86peL4HKwMXcQQtEa6lmC8FuD9XFt6gkNR0B/Mu5PE=
github.com/projectdiscovery/gozero v0.0.2 h1:8fJeaCjxL9tpm33uG/RsCQs6HGM/NE6eA3cjkilRQ+E=
From b84488e1cd60451b15f18fd1ebd10a41aca5dbf4 Mon Sep 17 00:00:00 2001
From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com>
Date: Mon, 5 Aug 2024 06:12:23 +0000
Subject: [PATCH 128/283] chore(deps): bump github.com/projectdiscovery/goflags
Bumps [github.com/projectdiscovery/goflags](https://github.com/projectdiscovery/goflags) from 0.1.59 to 0.1.62.
- [Release notes](https://github.com/projectdiscovery/goflags/releases)
- [Commits](https://github.com/projectdiscovery/goflags/compare/v0.1.59...v0.1.62)
---
updated-dependencies:
- dependency-name: github.com/projectdiscovery/goflags
dependency-type: direct:production
update-type: version-update:semver-patch
...
Signed-off-by: dependabot[bot]
---
go.mod | 2 +-
go.sum | 4 ++--
2 files changed, 3 insertions(+), 3 deletions(-)
diff --git a/go.mod b/go.mod
index 73b226ef8d..7fc76a19d8 100644
--- a/go.mod
+++ b/go.mod
@@ -82,7 +82,7 @@ require (
github.com/projectdiscovery/dsl v0.1.8
github.com/projectdiscovery/fasttemplate v0.0.2
github.com/projectdiscovery/go-smb2 v0.0.0-20240129202741-052cc450c6cb
- github.com/projectdiscovery/goflags v0.1.59
+ github.com/projectdiscovery/goflags v0.1.62
github.com/projectdiscovery/gologger v1.1.18
github.com/projectdiscovery/gostruct v0.0.2
github.com/projectdiscovery/gozero v0.0.2
diff --git a/go.sum b/go.sum
index a525d15be5..f9e7fbe640 100644
--- a/go.sum
+++ b/go.sum
@@ -842,8 +842,8 @@ github.com/projectdiscovery/freeport v0.0.5 h1:jnd3Oqsl4S8n0KuFkE5Hm8WGDP24ITBvm
github.com/projectdiscovery/freeport v0.0.5/go.mod h1:PY0bxSJ34HVy67LHIeF3uIutiCSDwOqKD8ruBkdiCwE=
github.com/projectdiscovery/go-smb2 v0.0.0-20240129202741-052cc450c6cb h1:rutG906Drtbpz4DwU5mhGIeOhRcktDH4cGQitGUMAsg=
github.com/projectdiscovery/go-smb2 v0.0.0-20240129202741-052cc450c6cb/go.mod h1:FLjF1DmZ+POoGEiIQdWuYVwS++C/GwpX8YaCsTSm1RY=
-github.com/projectdiscovery/goflags v0.1.59 h1:rFScpC57L0Ln0dJmBtstyxHUH7K9H4UmHACXcSkdqqk=
-github.com/projectdiscovery/goflags v0.1.59/go.mod h1:dj67QGp/D81WRLtzU0HsWR20zgoGZ0cnk3Wbt9xJcuo=
+github.com/projectdiscovery/goflags v0.1.62 h1:UmzKJQT+1UyqT1cZDmb3vZ8/IGhQ7LTsWfdqVcAGoJc=
+github.com/projectdiscovery/goflags v0.1.62/go.mod h1:d1/D8GaTDoV332ABwceUcY1ffKODaYFlGP0Oriq3wfk=
github.com/projectdiscovery/gologger v1.1.18 h1:fux2S/LXYe18/w4PylAFpEFg+EApNCuyS85x25nDE98=
github.com/projectdiscovery/gologger v1.1.18/go.mod h1:IgjKgVLfVGUovxlC6WJcXK0qvo/tNZ1P0+r6miQqyRk=
github.com/projectdiscovery/gostruct v0.0.2 h1:s8gP8ApugGM4go1pA+sVlPDXaWqNP5BBDDSv7VEdG1M=
From de29517572ff2f9968cde211c9c7bf012ba24d99 Mon Sep 17 00:00:00 2001
From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com>
Date: Mon, 5 Aug 2024 06:12:39 +0000
Subject: [PATCH 129/283] chore(deps): bump github.com/projectdiscovery/rawhttp
Bumps [github.com/projectdiscovery/rawhttp](https://github.com/projectdiscovery/rawhttp) from 0.1.57 to 0.1.60.
- [Release notes](https://github.com/projectdiscovery/rawhttp/releases)
- [Commits](https://github.com/projectdiscovery/rawhttp/compare/v0.1.57...v0.1.60)
---
updated-dependencies:
- dependency-name: github.com/projectdiscovery/rawhttp
dependency-type: direct:production
update-type: version-update:semver-patch
...
Signed-off-by: dependabot[bot]
---
go.mod | 6 +++---
go.sum | 12 ++++++------
2 files changed, 9 insertions(+), 9 deletions(-)
diff --git a/go.mod b/go.mod
index 73b226ef8d..2a352cd251 100644
--- a/go.mod
+++ b/go.mod
@@ -20,11 +20,11 @@ require (
github.com/olekukonko/tablewriter v0.0.5
github.com/pkg/errors v0.9.1
github.com/projectdiscovery/clistats v0.0.20
- github.com/projectdiscovery/fastdialer v0.2.1
+ github.com/projectdiscovery/fastdialer v0.2.2
github.com/projectdiscovery/hmap v0.0.51
github.com/projectdiscovery/interactsh v1.2.0
- github.com/projectdiscovery/rawhttp v0.1.57
- github.com/projectdiscovery/retryabledns v1.0.68
+ github.com/projectdiscovery/rawhttp v0.1.60
+ github.com/projectdiscovery/retryabledns v1.0.69
github.com/projectdiscovery/retryablehttp-go v1.0.71
github.com/projectdiscovery/yamldoc-go v1.0.4
github.com/remeh/sizedwaitgroup v1.0.0
diff --git a/go.sum b/go.sum
index a525d15be5..caafc6aaa3 100644
--- a/go.sum
+++ b/go.sum
@@ -834,8 +834,8 @@ github.com/projectdiscovery/clistats v0.0.20 h1:5jO5SLiRJ7f0nDV0ndBNmBeesbROouPo
github.com/projectdiscovery/clistats v0.0.20/go.mod h1:GJ2av0KnOvK0AISQnP8hyDclYIji1LVkx2l0pwnzAu4=
github.com/projectdiscovery/dsl v0.1.8 h1:ulz+o097XsYgqP4QXaAQhVSkoeD2ZPWd29wX3CTodUA=
github.com/projectdiscovery/dsl v0.1.8/go.mod h1:AYJS2WQ/q0smr2v4pEJTg4DPe6k56KFKR7UFXvzNz/4=
-github.com/projectdiscovery/fastdialer v0.2.1 h1:or3QuGW1jlZKi+IRkwxShSAG/hgR+yamd52RqjaZ28Q=
-github.com/projectdiscovery/fastdialer v0.2.1/go.mod h1:FGPJZIPzAfR7SyDCPTsftaf61lGOqIjrJpwo2IgkNpg=
+github.com/projectdiscovery/fastdialer v0.2.2 h1:AiUT4jIbS6JyM2QBQH9f+ZFBgvOVHzQG5K+M2FCA5jA=
+github.com/projectdiscovery/fastdialer v0.2.2/go.mod h1:QKrkTQiUhedqxqR65hLeN+FTbkul50TzMxgkk5Va+Uk=
github.com/projectdiscovery/fasttemplate v0.0.2 h1:h2cISk5xDhlJEinlBQS6RRx0vOlOirB2y3Yu4PJzpiA=
github.com/projectdiscovery/fasttemplate v0.0.2/go.mod h1:XYWWVMxnItd+r0GbjA1GCsUopMw1/XusuQxdyAIHMCw=
github.com/projectdiscovery/freeport v0.0.5 h1:jnd3Oqsl4S8n0KuFkE5Hm8WGDP24ITBvmyw5pFTHS8Q=
@@ -868,12 +868,12 @@ github.com/projectdiscovery/networkpolicy v0.0.9 h1:IrlDoYZagNNO8y+7iZeHT8k5izE+
github.com/projectdiscovery/networkpolicy v0.0.9/go.mod h1:XFJ2Lnv8BE/ziQCFjBHMsH1w6VmkPiQtk+NlBpdMU7M=
github.com/projectdiscovery/ratelimit v0.0.49 h1:PYatMp8g5OuoFsZOA90e48nLd2vB6a4Tw0FZ8h9zqkQ=
github.com/projectdiscovery/ratelimit v0.0.49/go.mod h1:Xi0LTMHg4HQlmCZFzRBIhRW6N+QW5RxQ8V/Qs+Vta4k=
-github.com/projectdiscovery/rawhttp v0.1.57 h1:2vCT2i1NSZbTBH+uUBrxOJjxDPKgIl2q6BGtQjs/Hko=
-github.com/projectdiscovery/rawhttp v0.1.57/go.mod h1:qtthyaU0k8eqcEdza1R/fTqwyxSK4BZ511ThxgkiQtE=
+github.com/projectdiscovery/rawhttp v0.1.60 h1:L57kAATGRuCfOp8Fs0ly3j2wlip7R5pco2RVfwqh/II=
+github.com/projectdiscovery/rawhttp v0.1.60/go.mod h1:M97w6lsphTHKeqLSio4Lw9K4KLO2riJMYAOqVMIZ3ck=
github.com/projectdiscovery/rdap v0.9.1-0.20221108103045-9865884d1917 h1:m03X4gBVSorSzvmm0bFa7gDV4QNSOWPL/fgZ4kTXBxk=
github.com/projectdiscovery/rdap v0.9.1-0.20221108103045-9865884d1917/go.mod h1:JxXtZC9e195awe7EynrcnBJmFoad/BNDzW9mzFkK8Sg=
-github.com/projectdiscovery/retryabledns v1.0.68 h1:EWWG7WsGTT0YvwIjHclIWSWgv4R29xMWShR6Yt5Z+pA=
-github.com/projectdiscovery/retryabledns v1.0.68/go.mod h1:72W9RwsHVRIGmtc4W6i6izVtYzKBTdnCE1VciqYM5Eg=
+github.com/projectdiscovery/retryabledns v1.0.69 h1:6vhhlfxPQoJI0j1enYSzyagyAZJNfGSg9TNloVsxEBQ=
+github.com/projectdiscovery/retryabledns v1.0.69/go.mod h1:5SDuGtXgL4W1z/9V+xnmHFYg03Fi1Ud/F2kqlPlAgfE=
github.com/projectdiscovery/retryablehttp-go v1.0.71 h1:yXPNShCOwoTz7dBSJsBhBh4g4ujX62XS/BrH/fL1VyE=
github.com/projectdiscovery/retryablehttp-go v1.0.71/go.mod h1:wY3T89EwcCKAw6iyMDvwzGPyL3d8TaBU80hnDErEKgM=
github.com/projectdiscovery/sarif v0.0.1 h1:C2Tyj0SGOKbCLgHrx83vaE6YkzXEVrMXYRGLkKCr/us=
From a7b815d561029c66b2f1284be74886c96d2cc54e Mon Sep 17 00:00:00 2001
From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com>
Date: Mon, 5 Aug 2024 06:40:21 +0000
Subject: [PATCH 130/283] chore(deps): bump
github.com/projectdiscovery/fastdialer
Bumps [github.com/projectdiscovery/fastdialer](https://github.com/projectdiscovery/fastdialer) from 0.2.1 to 0.2.3.
- [Release notes](https://github.com/projectdiscovery/fastdialer/releases)
- [Commits](https://github.com/projectdiscovery/fastdialer/compare/v0.2.1...v0.2.3)
---
updated-dependencies:
- dependency-name: github.com/projectdiscovery/fastdialer
dependency-type: direct:production
update-type: version-update:semver-patch
...
Signed-off-by: dependabot[bot]
---
go.mod | 4 ++--
go.sum | 8 ++++----
2 files changed, 6 insertions(+), 6 deletions(-)
diff --git a/go.mod b/go.mod
index 16c37c919c..0642eb604a 100644
--- a/go.mod
+++ b/go.mod
@@ -20,8 +20,8 @@ require (
github.com/olekukonko/tablewriter v0.0.5
github.com/pkg/errors v0.9.1
github.com/projectdiscovery/clistats v0.0.20
- github.com/projectdiscovery/fastdialer v0.2.2
- github.com/projectdiscovery/hmap v0.0.51
+ github.com/projectdiscovery/fastdialer v0.2.3
+ github.com/projectdiscovery/hmap v0.0.52
github.com/projectdiscovery/interactsh v1.2.0
github.com/projectdiscovery/rawhttp v0.1.60
github.com/projectdiscovery/retryabledns v1.0.69
diff --git a/go.sum b/go.sum
index e8e1d756e8..9ce53f77b1 100644
--- a/go.sum
+++ b/go.sum
@@ -834,8 +834,8 @@ github.com/projectdiscovery/clistats v0.0.20 h1:5jO5SLiRJ7f0nDV0ndBNmBeesbROouPo
github.com/projectdiscovery/clistats v0.0.20/go.mod h1:GJ2av0KnOvK0AISQnP8hyDclYIji1LVkx2l0pwnzAu4=
github.com/projectdiscovery/dsl v0.1.8 h1:ulz+o097XsYgqP4QXaAQhVSkoeD2ZPWd29wX3CTodUA=
github.com/projectdiscovery/dsl v0.1.8/go.mod h1:AYJS2WQ/q0smr2v4pEJTg4DPe6k56KFKR7UFXvzNz/4=
-github.com/projectdiscovery/fastdialer v0.2.2 h1:AiUT4jIbS6JyM2QBQH9f+ZFBgvOVHzQG5K+M2FCA5jA=
-github.com/projectdiscovery/fastdialer v0.2.2/go.mod h1:QKrkTQiUhedqxqR65hLeN+FTbkul50TzMxgkk5Va+Uk=
+github.com/projectdiscovery/fastdialer v0.2.3 h1:K03x5XEXGyVWxS2rtSR104E9kHF0aphN7kOCzbh8zv0=
+github.com/projectdiscovery/fastdialer v0.2.3/go.mod h1:a0BKvETrO1EAohUYp9gwtbbce0hKD1qGrTegyAUqyRo=
github.com/projectdiscovery/fasttemplate v0.0.2 h1:h2cISk5xDhlJEinlBQS6RRx0vOlOirB2y3Yu4PJzpiA=
github.com/projectdiscovery/fasttemplate v0.0.2/go.mod h1:XYWWVMxnItd+r0GbjA1GCsUopMw1/XusuQxdyAIHMCw=
github.com/projectdiscovery/freeport v0.0.5 h1:jnd3Oqsl4S8n0KuFkE5Hm8WGDP24ITBvmyw5pFTHS8Q=
@@ -850,8 +850,8 @@ github.com/projectdiscovery/gostruct v0.0.2 h1:s8gP8ApugGM4go1pA+sVlPDXaWqNP5BBD
github.com/projectdiscovery/gostruct v0.0.2/go.mod h1:H86peL4HKwMXcQQtEa6lmC8FuD9XFt6gkNR0B/Mu5PE=
github.com/projectdiscovery/gozero v0.0.2 h1:8fJeaCjxL9tpm33uG/RsCQs6HGM/NE6eA3cjkilRQ+E=
github.com/projectdiscovery/gozero v0.0.2/go.mod h1:d8bZvDWW07LWNYWrwjZ4OO1I0cpkfqaysyDfSs9ibK8=
-github.com/projectdiscovery/hmap v0.0.51 h1:xqbpRAJRHPMoS2uERkbWGObIO4bv+whe3PEk3h4lDEg=
-github.com/projectdiscovery/hmap v0.0.51/go.mod h1:vqdeWnNVMJYyIDytu+IdJDFg3wZdRVN83AKHR40RP6c=
+github.com/projectdiscovery/hmap v0.0.52 h1:Gm0aI0HDgsPW1+pq3PflO7PJ5cbOXUKt0DorlVsN3pI=
+github.com/projectdiscovery/hmap v0.0.52/go.mod h1:1YAS+8xYRIm+M8Qnlim8vw+U4rOPGLw5CqlirAVIPQ4=
github.com/projectdiscovery/httpx v1.6.6 h1:e9deBDrW2ILhuHjPYBAskIgEaLTJYbiBjC5FAHKuV4A=
github.com/projectdiscovery/httpx v1.6.6/go.mod h1:7kLxlw3gW6IJYwRbThI3rFkaQxJ/Z4zNeJtk408RnW0=
github.com/projectdiscovery/interactsh v1.2.0 h1:Al6jHiR+Usl9egYJDLJaWNHOcH8Rugk8gWMasc8Cmw8=
From 350fa4c10c43c0888561dfe1ee5fa473d5023f73 Mon Sep 17 00:00:00 2001
From: Peter Kasza
Date: Mon, 5 Aug 2024 11:53:21 +0200
Subject: [PATCH 131/283] fix: FileAuthProvider stores the same strategy for
each entry (#5474)
---
pkg/authprovider/file.go | 3 ++-
1 file changed, 2 insertions(+), 1 deletion(-)
diff --git a/pkg/authprovider/file.go b/pkg/authprovider/file.go
index 77f45e40fb..c20f85509d 100644
--- a/pkg/authprovider/file.go
+++ b/pkg/authprovider/file.go
@@ -51,7 +51,8 @@ func NewFileAuthProvider(path string, callback authx.LazyFetchSecret) (AuthProvi
// init initializes the file auth provider
func (f *FileAuthProvider) init() {
- for _, secret := range f.store.Secrets {
+ for _, _secret := range f.store.Secrets {
+ secret := _secret // allocate copy of pointer
if len(secret.DomainsRegex) > 0 {
for _, domain := range secret.DomainsRegex {
if f.compiled == nil {
From 543c8341737bfeb4dec0261496a5007d689c0cb7 Mon Sep 17 00:00:00 2001
From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com>
Date: Mon, 12 Aug 2024 05:12:30 +0000
Subject: [PATCH 132/283] chore(deps): bump github.com/projectdiscovery/httpx
from 1.6.6 to 1.6.7
Bumps [github.com/projectdiscovery/httpx](https://github.com/projectdiscovery/httpx) from 1.6.6 to 1.6.7.
- [Release notes](https://github.com/projectdiscovery/httpx/releases)
- [Changelog](https://github.com/projectdiscovery/httpx/blob/main/.goreleaser.yml)
- [Commits](https://github.com/projectdiscovery/httpx/compare/v1.6.6...v1.6.7)
---
updated-dependencies:
- dependency-name: github.com/projectdiscovery/httpx
dependency-type: direct:production
update-type: version-update:semver-patch
...
Signed-off-by: dependabot[bot]
---
go.mod | 2 +-
go.sum | 4 ++--
2 files changed, 3 insertions(+), 3 deletions(-)
diff --git a/go.mod b/go.mod
index 701289c076..276b111d1e 100644
--- a/go.mod
+++ b/go.mod
@@ -86,7 +86,7 @@ require (
github.com/projectdiscovery/gologger v1.1.18
github.com/projectdiscovery/gostruct v0.0.2
github.com/projectdiscovery/gozero v0.0.2
- github.com/projectdiscovery/httpx v1.6.6
+ github.com/projectdiscovery/httpx v1.6.7
github.com/projectdiscovery/mapcidr v1.1.34
github.com/projectdiscovery/n3iwf v0.0.0-20230523120440-b8cd232ff1f5
github.com/projectdiscovery/ratelimit v0.0.49
diff --git a/go.sum b/go.sum
index 25dd6b8add..571b7e1431 100644
--- a/go.sum
+++ b/go.sum
@@ -852,8 +852,8 @@ github.com/projectdiscovery/gozero v0.0.2 h1:8fJeaCjxL9tpm33uG/RsCQs6HGM/NE6eA3c
github.com/projectdiscovery/gozero v0.0.2/go.mod h1:d8bZvDWW07LWNYWrwjZ4OO1I0cpkfqaysyDfSs9ibK8=
github.com/projectdiscovery/hmap v0.0.52 h1:Gm0aI0HDgsPW1+pq3PflO7PJ5cbOXUKt0DorlVsN3pI=
github.com/projectdiscovery/hmap v0.0.52/go.mod h1:1YAS+8xYRIm+M8Qnlim8vw+U4rOPGLw5CqlirAVIPQ4=
-github.com/projectdiscovery/httpx v1.6.6 h1:e9deBDrW2ILhuHjPYBAskIgEaLTJYbiBjC5FAHKuV4A=
-github.com/projectdiscovery/httpx v1.6.6/go.mod h1:7kLxlw3gW6IJYwRbThI3rFkaQxJ/Z4zNeJtk408RnW0=
+github.com/projectdiscovery/httpx v1.6.7 h1:luJAtQ+iKpiyKKDpcWw1/fF1MaRX6JZ4R3P+ol5Uikk=
+github.com/projectdiscovery/httpx v1.6.7/go.mod h1:rqcuexBrb4v36ZyX0BHcHaovTurOJn2P65Tdt9cYdfE=
github.com/projectdiscovery/interactsh v1.2.0 h1:Al6jHiR+Usl9egYJDLJaWNHOcH8Rugk8gWMasc8Cmw8=
github.com/projectdiscovery/interactsh v1.2.0/go.mod h1:Wxt0fnzxsfrAZQQlpVrf3xMatP4OXZaZbjuDkIQKdYY=
github.com/projectdiscovery/ldapserver v1.0.2-0.20240219154113-dcc758ebc0cb h1:MGtI4oE12ruWv11ZlPXXd7hl/uAaQZrFvrIDYDeVMd8=
From 1643f51839ab30c3fb60088dcf219366e44482b1 Mon Sep 17 00:00:00 2001
From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com>
Date: Mon, 12 Aug 2024 05:12:34 +0000
Subject: [PATCH 133/283] chore(deps): bump
github.com/projectdiscovery/wappalyzergo
Bumps [github.com/projectdiscovery/wappalyzergo](https://github.com/projectdiscovery/wappalyzergo) from 0.1.13 to 0.1.14.
- [Release notes](https://github.com/projectdiscovery/wappalyzergo/releases)
- [Commits](https://github.com/projectdiscovery/wappalyzergo/compare/v0.1.13...v0.1.14)
---
updated-dependencies:
- dependency-name: github.com/projectdiscovery/wappalyzergo
dependency-type: direct:production
update-type: version-update:semver-patch
...
Signed-off-by: dependabot[bot]
---
go.mod | 2 +-
go.sum | 4 ++--
2 files changed, 3 insertions(+), 3 deletions(-)
diff --git a/go.mod b/go.mod
index 701289c076..b29b065ea1 100644
--- a/go.mod
+++ b/go.mod
@@ -96,7 +96,7 @@ require (
github.com/projectdiscovery/uncover v1.0.9
github.com/projectdiscovery/useragent v0.0.60
github.com/projectdiscovery/utils v0.2.3
- github.com/projectdiscovery/wappalyzergo v0.1.13
+ github.com/projectdiscovery/wappalyzergo v0.1.14
github.com/redis/go-redis/v9 v9.1.0
github.com/seh-msft/burpxml v1.0.1
github.com/stretchr/testify v1.9.0
diff --git a/go.sum b/go.sum
index 25dd6b8add..a523588428 100644
--- a/go.sum
+++ b/go.sum
@@ -888,8 +888,8 @@ github.com/projectdiscovery/useragent v0.0.60 h1:qDU1rwA+XOKmSqp7yoijAN4PuvLQc2Z
github.com/projectdiscovery/useragent v0.0.60/go.mod h1:05IDiJEy2dWl3x6dnsWtJYPwT40oWha144Us7+Fwr6w=
github.com/projectdiscovery/utils v0.2.3 h1:rkambl0EoTF/y6DpjCfSwcVUFdkAeVOtYkK3lX6InCY=
github.com/projectdiscovery/utils v0.2.3/go.mod h1:eGuuQ5Acekg47WsFS1Q9Qxw8+vI6IxwqIQSAplBBG0c=
-github.com/projectdiscovery/wappalyzergo v0.1.13 h1:tQ/pjdW4w2NHqUiFlNe/dULrPH8A9GOfnmplDzEOp3M=
-github.com/projectdiscovery/wappalyzergo v0.1.13/go.mod h1:/hzgxkBFTMe2wDbA93nFfoMjULw7/vIZ9QPSAnCgUa8=
+github.com/projectdiscovery/wappalyzergo v0.1.14 h1:nt1IM4RUmqeymsXk4h6BsZbKDoS2hjFvPkT2GaI1rz4=
+github.com/projectdiscovery/wappalyzergo v0.1.14/go.mod h1:/hzgxkBFTMe2wDbA93nFfoMjULw7/vIZ9QPSAnCgUa8=
github.com/projectdiscovery/yamldoc-go v1.0.4 h1:eZoESapnMw6WAHiVgRwNqvbJEfNHEH148uthhFbG5jE=
github.com/projectdiscovery/yamldoc-go v1.0.4/go.mod h1:8PIPRcUD55UbtQdcfFR1hpIGRWG0P7alClXNGt1TBik=
github.com/prometheus/client_golang v0.9.1/go.mod h1:7SWBe2y4D6OKWSNQJUaRYU/AaXPKyh/dDVn+NZz0KFw=
From 0b90504a0d4efa0341a9edcc662dfbae48aab94e Mon Sep 17 00:00:00 2001
From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com>
Date: Mon, 12 Aug 2024 05:39:17 +0000
Subject: [PATCH 134/283] chore(deps): bump
github.com/projectdiscovery/useragent
Bumps [github.com/projectdiscovery/useragent](https://github.com/projectdiscovery/useragent) from 0.0.60 to 0.0.65.
- [Release notes](https://github.com/projectdiscovery/useragent/releases)
- [Commits](https://github.com/projectdiscovery/useragent/compare/v0.0.60...v0.0.65)
---
updated-dependencies:
- dependency-name: github.com/projectdiscovery/useragent
dependency-type: direct:production
update-type: version-update:semver-patch
...
Signed-off-by: dependabot[bot]
---
go.mod | 4 ++--
go.sum | 8 ++++----
2 files changed, 6 insertions(+), 6 deletions(-)
diff --git a/go.mod b/go.mod
index b29b065ea1..47eb725239 100644
--- a/go.mod
+++ b/go.mod
@@ -94,8 +94,8 @@ require (
github.com/projectdiscovery/sarif v0.0.1
github.com/projectdiscovery/tlsx v1.1.6
github.com/projectdiscovery/uncover v1.0.9
- github.com/projectdiscovery/useragent v0.0.60
- github.com/projectdiscovery/utils v0.2.3
+ github.com/projectdiscovery/useragent v0.0.65
+ github.com/projectdiscovery/utils v0.2.4
github.com/projectdiscovery/wappalyzergo v0.1.14
github.com/redis/go-redis/v9 v9.1.0
github.com/seh-msft/burpxml v1.0.1
diff --git a/go.sum b/go.sum
index a523588428..b1ee8e63ea 100644
--- a/go.sum
+++ b/go.sum
@@ -884,10 +884,10 @@ github.com/projectdiscovery/tlsx v1.1.6 h1:iw2zwKbd2+kRQ8J1G4dLmS0CLyemd/tKz1Uzc
github.com/projectdiscovery/tlsx v1.1.6/go.mod h1:s7SRRFdrwIZBK/RXXZi4CR/CubqFSvp8h5Bk1srEZIo=
github.com/projectdiscovery/uncover v1.0.9 h1:s5RbkD/V4r8QcPkys4gTTqMuRSgXq0JprejqLSopN9Y=
github.com/projectdiscovery/uncover v1.0.9/go.mod h1:2PUF3SpB5QNIJ8epaB2xbRzkPaxEAWRDm3Ir2ijt81U=
-github.com/projectdiscovery/useragent v0.0.60 h1:qDU1rwA+XOKmSqp7yoijAN4PuvLQc2ZvuaoWH7QIkuc=
-github.com/projectdiscovery/useragent v0.0.60/go.mod h1:05IDiJEy2dWl3x6dnsWtJYPwT40oWha144Us7+Fwr6w=
-github.com/projectdiscovery/utils v0.2.3 h1:rkambl0EoTF/y6DpjCfSwcVUFdkAeVOtYkK3lX6InCY=
-github.com/projectdiscovery/utils v0.2.3/go.mod h1:eGuuQ5Acekg47WsFS1Q9Qxw8+vI6IxwqIQSAplBBG0c=
+github.com/projectdiscovery/useragent v0.0.65 h1:x78ZwWdqpzokOHxLITUXvq+ljkTKc19z3ILGtoV1N70=
+github.com/projectdiscovery/useragent v0.0.65/go.mod h1:deOP8YLJU6SCzM8k+K8PjkcOF4Ux0spqyO4ODZGIT4A=
+github.com/projectdiscovery/utils v0.2.4 h1:CHnlt2la4jr8TeL7ZK7UhQItHY7DDXqIuLnnxyAJLDY=
+github.com/projectdiscovery/utils v0.2.4/go.mod h1:2Vx7geSrBfCPqknZywqbChQm8SE30mcyrlB5YsxEnhA=
github.com/projectdiscovery/wappalyzergo v0.1.14 h1:nt1IM4RUmqeymsXk4h6BsZbKDoS2hjFvPkT2GaI1rz4=
github.com/projectdiscovery/wappalyzergo v0.1.14/go.mod h1:/hzgxkBFTMe2wDbA93nFfoMjULw7/vIZ9QPSAnCgUa8=
github.com/projectdiscovery/yamldoc-go v1.0.4 h1:eZoESapnMw6WAHiVgRwNqvbJEfNHEH148uthhFbG5jE=
From f363b307d416adf42a2a012b80e95587a1a93a95 Mon Sep 17 00:00:00 2001
From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com>
Date: Mon, 12 Aug 2024 06:06:05 +0000
Subject: [PATCH 135/283] chore(deps): bump github.com/projectdiscovery/goflags
Bumps [github.com/projectdiscovery/goflags](https://github.com/projectdiscovery/goflags) from 0.1.62 to 0.1.63.
- [Release notes](https://github.com/projectdiscovery/goflags/releases)
- [Commits](https://github.com/projectdiscovery/goflags/compare/v0.1.62...v0.1.63)
---
updated-dependencies:
- dependency-name: github.com/projectdiscovery/goflags
dependency-type: direct:production
update-type: version-update:semver-patch
...
Signed-off-by: dependabot[bot]
---
go.mod | 2 +-
go.sum | 4 ++--
2 files changed, 3 insertions(+), 3 deletions(-)
diff --git a/go.mod b/go.mod
index b17f837267..718bfdc9f5 100644
--- a/go.mod
+++ b/go.mod
@@ -82,7 +82,7 @@ require (
github.com/projectdiscovery/dsl v0.1.8
github.com/projectdiscovery/fasttemplate v0.0.2
github.com/projectdiscovery/go-smb2 v0.0.0-20240129202741-052cc450c6cb
- github.com/projectdiscovery/goflags v0.1.62
+ github.com/projectdiscovery/goflags v0.1.63
github.com/projectdiscovery/gologger v1.1.18
github.com/projectdiscovery/gostruct v0.0.2
github.com/projectdiscovery/gozero v0.0.2
diff --git a/go.sum b/go.sum
index 52421fd638..e9f71ce5d0 100644
--- a/go.sum
+++ b/go.sum
@@ -842,8 +842,8 @@ github.com/projectdiscovery/freeport v0.0.5 h1:jnd3Oqsl4S8n0KuFkE5Hm8WGDP24ITBvm
github.com/projectdiscovery/freeport v0.0.5/go.mod h1:PY0bxSJ34HVy67LHIeF3uIutiCSDwOqKD8ruBkdiCwE=
github.com/projectdiscovery/go-smb2 v0.0.0-20240129202741-052cc450c6cb h1:rutG906Drtbpz4DwU5mhGIeOhRcktDH4cGQitGUMAsg=
github.com/projectdiscovery/go-smb2 v0.0.0-20240129202741-052cc450c6cb/go.mod h1:FLjF1DmZ+POoGEiIQdWuYVwS++C/GwpX8YaCsTSm1RY=
-github.com/projectdiscovery/goflags v0.1.62 h1:UmzKJQT+1UyqT1cZDmb3vZ8/IGhQ7LTsWfdqVcAGoJc=
-github.com/projectdiscovery/goflags v0.1.62/go.mod h1:d1/D8GaTDoV332ABwceUcY1ffKODaYFlGP0Oriq3wfk=
+github.com/projectdiscovery/goflags v0.1.63 h1:xy3nqpzmSFEmG0LKTv3xLcp9LMbvbFkLZcjcnFe3xR4=
+github.com/projectdiscovery/goflags v0.1.63/go.mod h1:ptFbCm+5RmSHTSG4oniwBRRHJlguBTOOfMUfezeGbZ0=
github.com/projectdiscovery/gologger v1.1.18 h1:fux2S/LXYe18/w4PylAFpEFg+EApNCuyS85x25nDE98=
github.com/projectdiscovery/gologger v1.1.18/go.mod h1:IgjKgVLfVGUovxlC6WJcXK0qvo/tNZ1P0+r6miQqyRk=
github.com/projectdiscovery/gostruct v0.0.2 h1:s8gP8ApugGM4go1pA+sVlPDXaWqNP5BBDDSv7VEdG1M=
From 9019e36be519acbd6cd541f80fef8dc4c33ee692 Mon Sep 17 00:00:00 2001
From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com>
Date: Mon, 12 Aug 2024 06:06:06 +0000
Subject: [PATCH 136/283] chore(deps): bump github.com/projectdiscovery/hmap
from 0.0.52 to 0.0.54
Bumps [github.com/projectdiscovery/hmap](https://github.com/projectdiscovery/hmap) from 0.0.52 to 0.0.54.
- [Release notes](https://github.com/projectdiscovery/hmap/releases)
- [Commits](https://github.com/projectdiscovery/hmap/compare/v0.0.52...v0.0.54)
---
updated-dependencies:
- dependency-name: github.com/projectdiscovery/hmap
dependency-type: direct:production
update-type: version-update:semver-patch
...
Signed-off-by: dependabot[bot]
---
go.mod | 2 +-
go.sum | 4 ++--
2 files changed, 3 insertions(+), 3 deletions(-)
diff --git a/go.mod b/go.mod
index b17f837267..8f8b5bcb2d 100644
--- a/go.mod
+++ b/go.mod
@@ -21,7 +21,7 @@ require (
github.com/pkg/errors v0.9.1
github.com/projectdiscovery/clistats v0.0.20
github.com/projectdiscovery/fastdialer v0.2.3
- github.com/projectdiscovery/hmap v0.0.52
+ github.com/projectdiscovery/hmap v0.0.54
github.com/projectdiscovery/interactsh v1.2.0
github.com/projectdiscovery/rawhttp v0.1.60
github.com/projectdiscovery/retryabledns v1.0.69
diff --git a/go.sum b/go.sum
index 52421fd638..2ebc4095f8 100644
--- a/go.sum
+++ b/go.sum
@@ -850,8 +850,8 @@ github.com/projectdiscovery/gostruct v0.0.2 h1:s8gP8ApugGM4go1pA+sVlPDXaWqNP5BBD
github.com/projectdiscovery/gostruct v0.0.2/go.mod h1:H86peL4HKwMXcQQtEa6lmC8FuD9XFt6gkNR0B/Mu5PE=
github.com/projectdiscovery/gozero v0.0.2 h1:8fJeaCjxL9tpm33uG/RsCQs6HGM/NE6eA3cjkilRQ+E=
github.com/projectdiscovery/gozero v0.0.2/go.mod h1:d8bZvDWW07LWNYWrwjZ4OO1I0cpkfqaysyDfSs9ibK8=
-github.com/projectdiscovery/hmap v0.0.52 h1:Gm0aI0HDgsPW1+pq3PflO7PJ5cbOXUKt0DorlVsN3pI=
-github.com/projectdiscovery/hmap v0.0.52/go.mod h1:1YAS+8xYRIm+M8Qnlim8vw+U4rOPGLw5CqlirAVIPQ4=
+github.com/projectdiscovery/hmap v0.0.54 h1:b3pdQZwCw4is3xiL2jBx7SJZcYaf/7vtozY7bjUzO/s=
+github.com/projectdiscovery/hmap v0.0.54/go.mod h1:j0oakxYOWEfk29wRq5gQgrCv1JnfAfzGaMsRWwEas80=
github.com/projectdiscovery/httpx v1.6.7 h1:luJAtQ+iKpiyKKDpcWw1/fF1MaRX6JZ4R3P+ol5Uikk=
github.com/projectdiscovery/httpx v1.6.7/go.mod h1:rqcuexBrb4v36ZyX0BHcHaovTurOJn2P65Tdt9cYdfE=
github.com/projectdiscovery/interactsh v1.2.0 h1:Al6jHiR+Usl9egYJDLJaWNHOcH8Rugk8gWMasc8Cmw8=
From 1e49cd9800a138388df6116659166b791486d3ca Mon Sep 17 00:00:00 2001
From: Dwi Siswanto
Date: Thu, 15 Aug 2024 20:15:42 +0700
Subject: [PATCH 137/283] ci: don't clean modules cache (#5519)
* chore(make): add `verify` command
Signed-off-by: Dwi Siswanto
* ci: remove hygiene (`clean -modcache`) with verify instead
Signed-off-by: Dwi Siswanto
---------
Signed-off-by: Dwi Siswanto
---
.github/workflows/build-test.yml | 6 ++----
.github/workflows/performance-test.yaml | 6 ++----
Makefile | 4 +++-
3 files changed, 7 insertions(+), 9 deletions(-)
diff --git a/.github/workflows/build-test.yml b/.github/workflows/build-test.yml
index 92e2b479cd..f4707b2baf 100644
--- a/.github/workflows/build-test.yml
+++ b/.github/workflows/build-test.yml
@@ -27,10 +27,8 @@ jobs:
with:
python-version: '3.10'
- - name: Go Mod hygiene
- run: |
- go clean -modcache
- go mod tidy
+ - name: Verify Go modules
+ run: make verify
- name: Build
run: go build .
diff --git a/.github/workflows/performance-test.yaml b/.github/workflows/performance-test.yaml
index 6baecfaff2..92f5714bf4 100644
--- a/.github/workflows/performance-test.yaml
+++ b/.github/workflows/performance-test.yaml
@@ -22,10 +22,8 @@ jobs:
- name: Set up Go
uses: projectdiscovery/actions/setup/go@v1
- - name: Go Mod hygine
- run: |
- go clean -modcache
- go mod tidy
+ - name: Verify Go modules
+ run: make verify
# Max GH exection time 6H => timeout after that
- name: Running performance with big list
diff --git a/Makefile b/Makefile
index e916b980a6..a3f60a64b3 100644
--- a/Makefile
+++ b/Makefile
@@ -11,7 +11,7 @@ ifneq ($(shell go env GOOS),darwin)
LDFLAGS := -extldflags "-static"
endif
-.PHONY: all build build-stats scan-charts docs test integration functional tidy devtools jsupdate ts fuzzplayground memogen dsl-docs
+.PHONY: all build build-stats scan-charts docs test integration functional tidy verify devtools jsupdate ts fuzzplayground memogen dsl-docs
all: build
build:
@@ -39,6 +39,8 @@ functional:
cd cmd/functional-test; bash run.sh
tidy:
$(GOMOD) tidy
+verify: tidy
+ $(GOMOD) verify
devtools:
$(GOBUILD) $(GOFLAGS) -ldflags '$(LDFLAGS)' -o "bindgen" pkg/js/devtools/bindgen/cmd/bindgen/main.go
$(GOBUILD) $(GOFLAGS) -ldflags '$(LDFLAGS)' -o "tsgen" pkg/js/devtools/tsgen/cmd/tsgen/main.go
From e0466e102c08969b4ac887920c8efeee6f6f42fd Mon Sep 17 00:00:00 2001
From: Dogan Can Bakir <65292895+dogancanbakir@users.noreply.github.com>
Date: Fri, 16 Aug 2024 09:12:38 +0300
Subject: [PATCH 138/283] redact output (#5463)
* redact output
* update regex
* redact matchet-at
---
README.md | 1 +
cmd/nuclei/main.go | 1 +
pkg/output/output.go | 17 +++++++++++++++++
pkg/types/types.go | 2 ++
4 files changed, 21 insertions(+)
diff --git a/README.md b/README.md
index 6f69782e05..b6217a9a72 100644
--- a/README.md
+++ b/README.md
@@ -179,6 +179,7 @@ OUTPUT:
-se, -sarif-export string file to export results in SARIF format
-je, -json-export string file to export results in JSON format
-jle, -jsonl-export string file to export results in JSONL(ine) format
+ -rd, -redact string[] redact given list of keys from query parameter, request header and body
CONFIGURATIONS:
-config string path to the nuclei configuration file
diff --git a/cmd/nuclei/main.go b/cmd/nuclei/main.go
index 2d2be0aa1a..945af907ea 100644
--- a/cmd/nuclei/main.go
+++ b/cmd/nuclei/main.go
@@ -272,6 +272,7 @@ on extensive configurability, massive extensibility and ease of use.`)
flagSet.StringVarP(&options.SarifExport, "sarif-export", "se", "", "file to export results in SARIF format"),
flagSet.StringVarP(&options.JSONExport, "json-export", "je", "", "file to export results in JSON format"),
flagSet.StringVarP(&options.JSONLExport, "jsonl-export", "jle", "", "file to export results in JSONL(ine) format"),
+ flagSet.StringSliceVarP(&options.Redact, "redact", "rd", nil, "redact given list of keys from query parameter, request header and body", goflags.CommaSeparatedStringSliceOptions),
)
flagSet.CreateGroup("configs", "Configurations",
diff --git a/pkg/output/output.go b/pkg/output/output.go
index 8044c0aed9..9f68122b0f 100644
--- a/pkg/output/output.go
+++ b/pkg/output/output.go
@@ -71,6 +71,7 @@ type StandardWriter struct {
omitTemplate bool
DisableStdout bool
AddNewLinesOutputFile bool // by default this is only done for stdout
+ KeysToRedact []string
}
var decolorizerRegex = regexp.MustCompile(`\x1B\[[0-9;]*[a-zA-Z]`)
@@ -253,6 +254,7 @@ func NewStandardWriter(options *types.Options) (*StandardWriter, error) {
storeResponse: options.StoreResponse,
storeResponseDir: options.StoreResponseDir,
omitTemplate: options.OmitTemplate,
+ KeysToRedact: options.Redact,
}
return writer, nil
}
@@ -264,6 +266,13 @@ func (w *StandardWriter) Write(event *ResultEvent) error {
event.Template, event.TemplateURL = utils.TemplatePathURL(types.ToString(event.TemplatePath), types.ToString(event.TemplateID))
}
+ if len(w.KeysToRedact) > 0 {
+ event.Request = redactKeys(event.Request, w.KeysToRedact)
+ event.Response = redactKeys(event.Response, w.KeysToRedact)
+ event.CURLCommand = redactKeys(event.CURLCommand, w.KeysToRedact)
+ event.Matched = redactKeys(event.Matched, w.KeysToRedact)
+ }
+
event.Timestamp = time.Now()
var data []byte
@@ -302,6 +311,14 @@ func (w *StandardWriter) Write(event *ResultEvent) error {
return nil
}
+func redactKeys(data string, keysToRedact []string) string {
+ for _, key := range keysToRedact {
+ keyPattern := regexp.MustCompile(fmt.Sprintf(`(?i)(%s\s*[:=]\s*["']?)[^"'\r\n&]+(["'\r\n]?)`, regexp.QuoteMeta(key)))
+ data = keyPattern.ReplaceAllString(data, `$1***$2`)
+ }
+ return data
+}
+
// JSONLogRequest is a trace/error log request written to file
type JSONLogRequest struct {
Template string `json:"template"`
diff --git a/pkg/types/types.go b/pkg/types/types.go
index 9d05a0b742..32c7f6f6a2 100644
--- a/pkg/types/types.go
+++ b/pkg/types/types.go
@@ -220,6 +220,8 @@ type Options struct {
JSONExport string
// JSONLExport is the file to export JSONL output format to
JSONLExport string
+ // Redact redacts given keys in
+ Redact goflags.StringSlice
// EnableProgressBar enables progress bar
EnableProgressBar bool
// TemplateDisplay displays the template contents
From 2609d2d135e474b567c14decad1be9f57ab8820b Mon Sep 17 00:00:00 2001
From: Ramana Reddy <90540245+RamanaReddy0M@users.noreply.github.com>
Date: Fri, 16 Aug 2024 11:59:15 +0530
Subject: [PATCH 139/283] feat: add support for multiple auth strategies per
target from secrets file (#5500)
---
pkg/authprovider/file.go | 78 +++++++++++++++++------------
pkg/authprovider/interface.go | 6 +--
pkg/authprovider/multi.go | 6 +--
pkg/protocols/http/build_request.go | 16 +++---
4 files changed, 61 insertions(+), 45 deletions(-)
diff --git a/pkg/authprovider/file.go b/pkg/authprovider/file.go
index c20f85509d..3a32a94fe4 100644
--- a/pkg/authprovider/file.go
+++ b/pkg/authprovider/file.go
@@ -16,8 +16,8 @@ import (
type FileAuthProvider struct {
Path string
store *authx.Authx
- compiled map[*regexp.Regexp]authx.AuthStrategy
- domains map[string]authx.AuthStrategy
+ compiled map[*regexp.Regexp][]authx.AuthStrategy
+ domains map[string][]authx.AuthStrategy
}
// NewFileAuthProvider creates a new file based auth provider
@@ -56,25 +56,31 @@ func (f *FileAuthProvider) init() {
if len(secret.DomainsRegex) > 0 {
for _, domain := range secret.DomainsRegex {
if f.compiled == nil {
- f.compiled = make(map[*regexp.Regexp]authx.AuthStrategy)
+ f.compiled = make(map[*regexp.Regexp][]authx.AuthStrategy)
}
compiled, err := regexp.Compile(domain)
if err != nil {
continue
}
- f.compiled[compiled] = secret.GetStrategy()
+
+ if ss, ok := f.compiled[compiled]; ok {
+ f.compiled[compiled] = append(ss, secret.GetStrategy())
+ } else {
+ f.compiled[compiled] = []authx.AuthStrategy{secret.GetStrategy()}
+ }
}
}
for _, domain := range secret.Domains {
if f.domains == nil {
- f.domains = make(map[string]authx.AuthStrategy)
+ f.domains = make(map[string][]authx.AuthStrategy)
}
- f.domains[strings.TrimSpace(domain)] = secret.GetStrategy()
- if strings.HasSuffix(domain, ":80") {
- f.domains[strings.TrimSuffix(domain, ":80")] = secret.GetStrategy()
- }
- if strings.HasSuffix(domain, ":443") {
- f.domains[strings.TrimSuffix(domain, ":443")] = secret.GetStrategy()
+ domain = strings.TrimSpace(domain)
+ domain = strings.TrimSuffix(domain, ":80")
+ domain = strings.TrimSuffix(domain, ":443")
+ if ss, ok := f.domains[domain]; ok {
+ f.domains[domain] = append(ss, secret.GetStrategy())
+ } else {
+ f.domains[domain] = []authx.AuthStrategy{secret.GetStrategy()}
}
}
}
@@ -82,32 +88,38 @@ func (f *FileAuthProvider) init() {
if len(dynamic.DomainsRegex) > 0 {
for _, domain := range dynamic.DomainsRegex {
if f.compiled == nil {
- f.compiled = make(map[*regexp.Regexp]authx.AuthStrategy)
+ f.compiled = make(map[*regexp.Regexp][]authx.AuthStrategy)
}
compiled, err := regexp.Compile(domain)
if err != nil {
continue
}
- f.compiled[compiled] = &authx.DynamicAuthStrategy{Dynamic: dynamic}
+ if ss, ok := f.compiled[compiled]; !ok {
+ f.compiled[compiled] = []authx.AuthStrategy{&authx.DynamicAuthStrategy{Dynamic: dynamic}}
+ } else {
+ f.compiled[compiled] = append(ss, &authx.DynamicAuthStrategy{Dynamic: dynamic})
+ }
}
}
for _, domain := range dynamic.Domains {
if f.domains == nil {
- f.domains = make(map[string]authx.AuthStrategy)
+ f.domains = make(map[string][]authx.AuthStrategy)
}
- f.domains[strings.TrimSpace(domain)] = &authx.DynamicAuthStrategy{Dynamic: dynamic}
- if strings.HasSuffix(domain, ":80") {
- f.domains[strings.TrimSuffix(domain, ":80")] = &authx.DynamicAuthStrategy{Dynamic: dynamic}
- }
- if strings.HasSuffix(domain, ":443") {
- f.domains[strings.TrimSuffix(domain, ":443")] = &authx.DynamicAuthStrategy{Dynamic: dynamic}
+ domain = strings.TrimSpace(domain)
+ domain = strings.TrimSuffix(domain, ":80")
+ domain = strings.TrimSuffix(domain, ":443")
+
+ if ss, ok := f.domains[domain]; !ok {
+ f.domains[domain] = []authx.AuthStrategy{&authx.DynamicAuthStrategy{Dynamic: dynamic}}
+ } else {
+ f.domains[domain] = append(ss, &authx.DynamicAuthStrategy{Dynamic: dynamic})
}
}
}
}
// LookupAddr looks up a given domain/address and returns appropriate auth strategy
-func (f *FileAuthProvider) LookupAddr(addr string) authx.AuthStrategy {
+func (f *FileAuthProvider) LookupAddr(addr string) []authx.AuthStrategy {
if strings.Contains(addr, ":") {
// default normalization for host:port
host, port, err := net.SplitHostPort(addr)
@@ -129,12 +141,12 @@ func (f *FileAuthProvider) LookupAddr(addr string) authx.AuthStrategy {
}
// LookupURL looks up a given URL and returns appropriate auth strategy
-func (f *FileAuthProvider) LookupURL(u *url.URL) authx.AuthStrategy {
+func (f *FileAuthProvider) LookupURL(u *url.URL) []authx.AuthStrategy {
return f.LookupAddr(u.Host)
}
// LookupURLX looks up a given URL and returns appropriate auth strategy
-func (f *FileAuthProvider) LookupURLX(u *urlutil.URL) authx.AuthStrategy {
+func (f *FileAuthProvider) LookupURLX(u *urlutil.URL) []authx.AuthStrategy {
return f.LookupAddr(u.Host)
}
@@ -151,17 +163,21 @@ func (f *FileAuthProvider) GetTemplatePaths() []string {
// PreFetchSecrets pre-fetches the secrets from the auth provider
func (f *FileAuthProvider) PreFetchSecrets() error {
- for _, s := range f.domains {
- if val, ok := s.(*authx.DynamicAuthStrategy); ok {
- if err := val.Dynamic.Fetch(false); err != nil {
- return err
+ for _, ss := range f.domains {
+ for _, s := range ss {
+ if val, ok := s.(*authx.DynamicAuthStrategy); ok {
+ if err := val.Dynamic.Fetch(false); err != nil {
+ return err
+ }
}
}
}
- for _, s := range f.compiled {
- if val, ok := s.(*authx.DynamicAuthStrategy); ok {
- if err := val.Dynamic.Fetch(false); err != nil {
- return err
+ for _, ss := range f.compiled {
+ for _, s := range ss {
+ if val, ok := s.(*authx.DynamicAuthStrategy); ok {
+ if err := val.Dynamic.Fetch(false); err != nil {
+ return err
+ }
}
}
}
diff --git a/pkg/authprovider/interface.go b/pkg/authprovider/interface.go
index b21668fceb..ea1bba7ced 100644
--- a/pkg/authprovider/interface.go
+++ b/pkg/authprovider/interface.go
@@ -22,13 +22,13 @@ var (
type AuthProvider interface {
// LookupAddr looks up a given domain/address and returns appropriate auth strategy
// for it (accepted inputs are scanme.sh or scanme.sh:443)
- LookupAddr(string) authx.AuthStrategy
+ LookupAddr(string) []authx.AuthStrategy
// LookupURL looks up a given URL and returns appropriate auth strategy
// it accepts a valid url struct and returns the auth strategy
- LookupURL(*url.URL) authx.AuthStrategy
+ LookupURL(*url.URL) []authx.AuthStrategy
// LookupURLX looks up a given URL and returns appropriate auth strategy
// it accepts pd url struct (i.e urlutil.URL) and returns the auth strategy
- LookupURLX(*urlutil.URL) authx.AuthStrategy
+ LookupURLX(*urlutil.URL) []authx.AuthStrategy
// GetTemplatePaths returns the template path for the auth provider
// that will be used for dynamic secret fetching
GetTemplatePaths() []string
diff --git a/pkg/authprovider/multi.go b/pkg/authprovider/multi.go
index 2e9b19df8c..d059a75922 100644
--- a/pkg/authprovider/multi.go
+++ b/pkg/authprovider/multi.go
@@ -19,7 +19,7 @@ func NewMultiAuthProvider(providers ...AuthProvider) AuthProvider {
return &MultiAuthProvider{Providers: providers}
}
-func (m *MultiAuthProvider) LookupAddr(host string) authx.AuthStrategy {
+func (m *MultiAuthProvider) LookupAddr(host string) []authx.AuthStrategy {
for _, provider := range m.Providers {
strategy := provider.LookupAddr(host)
if strategy != nil {
@@ -29,7 +29,7 @@ func (m *MultiAuthProvider) LookupAddr(host string) authx.AuthStrategy {
return nil
}
-func (m *MultiAuthProvider) LookupURL(u *url.URL) authx.AuthStrategy {
+func (m *MultiAuthProvider) LookupURL(u *url.URL) []authx.AuthStrategy {
for _, provider := range m.Providers {
strategy := provider.LookupURL(u)
if strategy != nil {
@@ -39,7 +39,7 @@ func (m *MultiAuthProvider) LookupURL(u *url.URL) authx.AuthStrategy {
return nil
}
-func (m *MultiAuthProvider) LookupURLX(u *urlutil.URL) authx.AuthStrategy {
+func (m *MultiAuthProvider) LookupURLX(u *urlutil.URL) []authx.AuthStrategy {
for _, provider := range m.Providers {
strategy := provider.LookupURLX(u)
if strategy != nil {
diff --git a/pkg/protocols/http/build_request.go b/pkg/protocols/http/build_request.go
index 18b75c72bd..1b046bffd8 100644
--- a/pkg/protocols/http/build_request.go
+++ b/pkg/protocols/http/build_request.go
@@ -90,9 +90,9 @@ func (g *generatedRequest) ApplyAuth(provider authprovider.AuthProvider) {
return
}
if g.request != nil {
- auth := provider.LookupURLX(g.request.URL)
- if auth != nil {
- auth.ApplyOnRR(g.request)
+ authStrategies := provider.LookupURLX(g.request.URL)
+ for _, strategy := range authStrategies {
+ strategy.ApplyOnRR(g.request)
}
}
if g.rawRequest != nil {
@@ -101,11 +101,11 @@ func (g *generatedRequest) ApplyAuth(provider authprovider.AuthProvider) {
gologger.Warning().Msgf("[authprovider] Could not parse URL %s: %s\n", g.rawRequest.FullURL, err)
return
}
- auth := provider.LookupURLX(parsed)
- if auth != nil {
- // here we need to apply it custom because we don't have a standard/official
- // rawhttp request format ( which we probably should have )
- g.rawRequest.ApplyAuthStrategy(auth)
+ authStrategies := provider.LookupURLX(parsed)
+ // here we need to apply it custom because we don't have a standard/official
+ // rawhttp request format ( which we probably should have )
+ for _, strategy := range authStrategies {
+ g.rawRequest.ApplyAuthStrategy(strategy)
}
}
}
From 2f7eea410df292ab18c657e8aa34240a07475c6a Mon Sep 17 00:00:00 2001
From: Ramana Reddy <90540245+RamanaReddy0M@users.noreply.github.com>
Date: Fri, 16 Aug 2024 13:27:26 +0530
Subject: [PATCH 140/283] Add `team-id` option (#5523)
* add team-id option
* fix dashboard url when uploading to team
---------
Co-authored-by: Tarun Koyalwar
---
cmd/nuclei/main.go | 2 ++
internal/pdcp/utils.go | 10 +++++++++-
internal/pdcp/writer.go | 25 ++++++++++++++++++-------
internal/runner/runner.go | 3 +++
pkg/types/types.go | 2 ++
5 files changed, 34 insertions(+), 8 deletions(-)
diff --git a/cmd/nuclei/main.go b/cmd/nuclei/main.go
index 945af907ea..de29d6581e 100644
--- a/cmd/nuclei/main.go
+++ b/cmd/nuclei/main.go
@@ -12,6 +12,7 @@ import (
"strings"
"time"
+ _pdcp "github.com/projectdiscovery/nuclei/v3/internal/pdcp"
"github.com/projectdiscovery/utils/auth/pdcp"
"github.com/projectdiscovery/utils/env"
_ "github.com/projectdiscovery/utils/pprof"
@@ -418,6 +419,7 @@ on extensive configurability, massive extensibility and ease of use.`)
flagSet.CreateGroup("cloud", "Cloud",
flagSet.DynamicVar(&pdcpauth, "auth", "true", "configure projectdiscovery cloud (pdcp) api key"),
+ flagSet.StringVarP(&options.TeamID, "team-id", "tid", _pdcp.TeamIDEnv, "upload scan results to given team id (optional)"),
flagSet.BoolVarP(&options.EnableCloudUpload, "cloud-upload", "cup", false, "upload scan results to pdcp dashboard"),
flagSet.StringVarP(&options.ScanID, "scan-id", "sid", "", "upload scan results to existing scan id (optional)"),
flagSet.StringVarP(&options.ScanName, "scan-name", "sname", "", "scan name to set (optional)"),
diff --git a/internal/pdcp/utils.go b/internal/pdcp/utils.go
index 5d4fa4e1f0..3385d5cad5 100644
--- a/internal/pdcp/utils.go
+++ b/internal/pdcp/utils.go
@@ -5,9 +5,17 @@ import (
urlutil "github.com/projectdiscovery/utils/url"
)
-func getScanDashBoardURL(id string) string {
+func getScanDashBoardURL(id string, teamID string) string {
ux, _ := urlutil.Parse(pdcpauth.DashBoardURL)
ux.Path = "/scans/" + id
+ if ux.Params == nil {
+ ux.Params = urlutil.NewOrderedParams()
+ }
+ if teamID != "" {
+ ux.Params.Add("team_id", teamID)
+ } else {
+ ux.Params.Add("team_id", NoneTeamID)
+ }
ux.Update()
return ux.String()
}
diff --git a/internal/pdcp/writer.go b/internal/pdcp/writer.go
index 2197041137..aa2d5134d0 100644
--- a/internal/pdcp/writer.go
+++ b/internal/pdcp/writer.go
@@ -32,13 +32,14 @@ const (
MaxChunkSize = 4 * unitutils.Mega // 4 MB
xidRe = `^[a-z0-9]{20}$`
teamIDHeader = "X-Team-Id"
+ NoneTeamID = "none"
)
var (
xidRegex = regexp.MustCompile(xidRe)
_ output.Writer = &UploadWriter{}
// teamID if given
- teamID = env.GetEnvOrDefault("PDCP_TEAM_ID", "")
+ TeamIDEnv = env.GetEnvOrDefault("PDCP_TEAM_ID", NoneTeamID)
)
// UploadWriter is a writer that uploads its output to pdcp
@@ -53,6 +54,7 @@ type UploadWriter struct {
scanID string
scanName string
counter atomic.Int32
+ TeamID string
}
// NewUploadWriter creates a new upload writer
@@ -61,8 +63,9 @@ func NewUploadWriter(ctx context.Context, creds *pdcpauth.PDCPCredentials) (*Upl
return nil, fmt.Errorf("no credentials provided")
}
u := &UploadWriter{
- creds: creds,
- done: make(chan struct{}, 1),
+ creds: creds,
+ done: make(chan struct{}, 1),
+ TeamID: NoneTeamID,
}
var err error
reader, writer := io.Pipe()
@@ -110,6 +113,14 @@ func (u *UploadWriter) SetScanName(name string) {
u.scanName = name
}
+func (u *UploadWriter) SetTeamID(id string) {
+ if id == "" {
+ u.TeamID = NoneTeamID
+ } else {
+ u.TeamID = id
+ }
+}
+
func (u *UploadWriter) autoCommit(ctx context.Context, r *io.PipeReader) {
reader := bufio.NewReader(r)
ch := make(chan string, 4)
@@ -136,7 +147,7 @@ func (u *UploadWriter) autoCommit(ctx context.Context, r *io.PipeReader) {
if u.scanID == "" {
gologger.Verbose().Msgf("Scan results upload to cloud skipped, no results found to upload")
} else {
- gologger.Info().Msgf("%v Scan results uploaded to cloud, you can view scan results at %v", u.counter.Load(), getScanDashBoardURL(u.scanID))
+ gologger.Info().Msgf("%v Scan results uploaded to cloud, you can view scan results at %v", u.counter.Load(), getScanDashBoardURL(u.scanID, u.TeamID))
}
}()
// temporary buffer to store the results
@@ -189,7 +200,7 @@ func (u *UploadWriter) uploadChunk(buff *bytes.Buffer) error {
// if successful, reset the buffer
buff.Reset()
// log in verbose mode
- gologger.Warning().Msgf("Uploaded results chunk, you can view scan results at %v", getScanDashBoardURL(u.scanID))
+ gologger.Warning().Msgf("Uploaded results chunk, you can view scan results at %v", getScanDashBoardURL(u.scanID, u.TeamID))
return nil
}
@@ -248,8 +259,8 @@ func (u *UploadWriter) getRequest(bin []byte) (*retryablehttp.Request, error) {
req.URL.Update()
req.Header.Set(pdcpauth.ApiKeyHeaderName, u.creds.APIKey)
- if teamID != "" {
- req.Header.Set(teamIDHeader, teamID)
+ if u.TeamID != NoneTeamID && u.TeamID != "" {
+ req.Header.Set(teamIDHeader, u.TeamID)
}
req.Header.Set("Content-Type", "application/octet-stream")
req.Header.Set("Accept", "application/json")
diff --git a/internal/runner/runner.go b/internal/runner/runner.go
index bc436500af..bfb2bc64be 100644
--- a/internal/runner/runner.go
+++ b/internal/runner/runner.go
@@ -426,6 +426,9 @@ func (r *Runner) setupPDCPUpload(writer output.Writer) output.Writer {
if r.options.ScanName != "" {
uploadWriter.SetScanName(r.options.ScanName)
}
+ if r.options.TeamID != "" {
+ uploadWriter.SetTeamID(r.options.TeamID)
+ }
return output.NewMultiWriter(writer, uploadWriter)
}
diff --git a/pkg/types/types.go b/pkg/types/types.go
index 32c7f6f6a2..cab1aacf54 100644
--- a/pkg/types/types.go
+++ b/pkg/types/types.go
@@ -384,6 +384,8 @@ type Options struct {
ScanID string
// ScanName is the name of the scan to be uploaded
ScanName string
+ // TeamID is the team ID to use for cloud upload
+ TeamID string
// JsConcurrency is the number of concurrent js routines to run
JsConcurrency int
// SecretsFile is file containing secrets for nuclei
From c6e5bdd8570cdc04270779c2630da4a556d782b6 Mon Sep 17 00:00:00 2001
From: Tryfon Papatriantafyllou <67585616+trypa11@users.noreply.github.com>
Date: Fri, 16 Aug 2024 15:07:02 +0300
Subject: [PATCH 141/283] Fixing the server URL path for OpenAPI scanning
(#5504)
* fix_openAPI_serverURL_path
* Issue #5503
---
pkg/input/formats/openapi/generator.go | 14 ++++++++++++--
pkg/input/formats/swagger/swagger_test.go | 4 ++--
2 files changed, 14 insertions(+), 4 deletions(-)
diff --git a/pkg/input/formats/openapi/generator.go b/pkg/input/formats/openapi/generator.go
index 6f10c56d09..adcb26b2fa 100644
--- a/pkg/input/formats/openapi/generator.go
+++ b/pkg/input/formats/openapi/generator.go
@@ -72,19 +72,29 @@ func GenerateRequestsFromSchema(schema *openapi3.T, opts formats.InputFormatOpti
}
missingVarMap[param.Name] = struct{}{}
}
-
+
for _, serverURL := range schema.Servers {
pathURL := serverURL.URL
+ // Split the server URL into baseURL and serverPath
+ u, err := url.Parse(pathURL)
+ if err != nil {
+ return errors.Wrap(err, "could not parse server url")
+ }
+ baseURL := fmt.Sprintf("%s://%s", u.Scheme, u.Host)
+ serverPath := u.Path
for path, v := range schema.Paths.Map() {
// a path item can have parameters
ops := v.Operations()
requestPath := path
+ if serverPath != "" {
+ requestPath = serverPath + path
+ }
for method, ov := range ops {
if err := generateRequestsFromOp(&generateReqOptions{
requiredOnly: opts.RequiredOnly,
method: method,
- pathURL: pathURL,
+ pathURL: baseURL,
requestPath: requestPath,
op: ov,
schema: schema,
diff --git a/pkg/input/formats/swagger/swagger_test.go b/pkg/input/formats/swagger/swagger_test.go
index 601c20d94b..065ae78f63 100644
--- a/pkg/input/formats/swagger/swagger_test.go
+++ b/pkg/input/formats/swagger/swagger_test.go
@@ -27,8 +27,8 @@ func TestSwaggerAPIParser(t *testing.T) {
}
expectedURLs := []string{
- "https://localhost/users",
- "https://localhost/users/1?test=asc",
+ "https://localhost/v1/users",
+ "https://localhost/v1/users/1?test=asc",
}
require.ElementsMatch(t, gotMethodsToURLs, expectedURLs, "could not get swagger urls")
}
From 1293a34707198ff58445385ab404046e4b668c39 Mon Sep 17 00:00:00 2001
From: sandeep <8293321+ehsandeep@users.noreply.github.com>
Date: Fri, 16 Aug 2024 18:03:53 +0530
Subject: [PATCH 142/283] dep update
---
go.mod | 2 +-
go.sum | 4 ++--
2 files changed, 3 insertions(+), 3 deletions(-)
diff --git a/go.mod b/go.mod
index 0130e9d3af..0f729d5598 100644
--- a/go.mod
+++ b/go.mod
@@ -83,7 +83,7 @@ require (
github.com/projectdiscovery/fasttemplate v0.0.2
github.com/projectdiscovery/go-smb2 v0.0.0-20240129202741-052cc450c6cb
github.com/projectdiscovery/goflags v0.1.63
- github.com/projectdiscovery/gologger v1.1.18
+ github.com/projectdiscovery/gologger v1.1.19
github.com/projectdiscovery/gostruct v0.0.2
github.com/projectdiscovery/gozero v0.0.2
github.com/projectdiscovery/httpx v1.6.7
diff --git a/go.sum b/go.sum
index 9f4c82e8b5..3f8cef8e83 100644
--- a/go.sum
+++ b/go.sum
@@ -844,8 +844,8 @@ github.com/projectdiscovery/go-smb2 v0.0.0-20240129202741-052cc450c6cb h1:rutG90
github.com/projectdiscovery/go-smb2 v0.0.0-20240129202741-052cc450c6cb/go.mod h1:FLjF1DmZ+POoGEiIQdWuYVwS++C/GwpX8YaCsTSm1RY=
github.com/projectdiscovery/goflags v0.1.63 h1:xy3nqpzmSFEmG0LKTv3xLcp9LMbvbFkLZcjcnFe3xR4=
github.com/projectdiscovery/goflags v0.1.63/go.mod h1:ptFbCm+5RmSHTSG4oniwBRRHJlguBTOOfMUfezeGbZ0=
-github.com/projectdiscovery/gologger v1.1.18 h1:fux2S/LXYe18/w4PylAFpEFg+EApNCuyS85x25nDE98=
-github.com/projectdiscovery/gologger v1.1.18/go.mod h1:IgjKgVLfVGUovxlC6WJcXK0qvo/tNZ1P0+r6miQqyRk=
+github.com/projectdiscovery/gologger v1.1.19 h1:b7cU32XuDrDiwhr7hlDeE6mfj/nENBtHEohe51txJCE=
+github.com/projectdiscovery/gologger v1.1.19/go.mod h1:DbeKwx9IEfcvnclImX5gBlhIKUuOZwOM5itdpYXl+54=
github.com/projectdiscovery/gostruct v0.0.2 h1:s8gP8ApugGM4go1pA+sVlPDXaWqNP5BBDDSv7VEdG1M=
github.com/projectdiscovery/gostruct v0.0.2/go.mod h1:H86peL4HKwMXcQQtEa6lmC8FuD9XFt6gkNR0B/Mu5PE=
github.com/projectdiscovery/gozero v0.0.2 h1:8fJeaCjxL9tpm33uG/RsCQs6HGM/NE6eA3cjkilRQ+E=
From 1af29f97a993d7e47968c2e509f0d1857b535401 Mon Sep 17 00:00:00 2001
From: Dwi Siswanto
Date: Fri, 16 Aug 2024 19:40:48 +0700
Subject: [PATCH 143/283] feat(http): add `skip-secret-file` field (#5522)
* feat(http): add `BypassSecretFile` field
Signed-off-by: Dwi Siswanto
* feat(http): conditionally apply auth strategies
Signed-off-by: Dwi Siswanto
* refactor(http): rename `BypassSecretFile` field to `SkipSecretFile`
Signed-off-by: Dwi Siswanto
---------
Signed-off-by: Dwi Siswanto
---
pkg/protocols/http/http.go | 4 ++++
pkg/protocols/http/request.go | 2 +-
2 files changed, 5 insertions(+), 1 deletion(-)
diff --git a/pkg/protocols/http/http.go b/pkg/protocols/http/http.go
index fc347ce88d..844bf8c579 100644
--- a/pkg/protocols/http/http.go
+++ b/pkg/protocols/http/http.go
@@ -147,6 +147,10 @@ type Request struct {
// - "AWS"
Signature SignatureTypeHolder `yaml:"signature,omitempty" json:"signature,omitempty" jsonschema:"title=signature is the http request signature method,description=Signature is the HTTP Request signature Method,enum=AWS"`
+ // description: |
+ // SkipSecretFile skips the authentication or authorization configured in the secret file.
+ SkipSecretFile bool `yaml:"skip-secret-file,omitempty" json:"skip-secret-file,omitempty" jsonschema:"title=bypass secret file,description=Skips the authentication or authorization configured in the secret file"`
+
// description: |
// CookieReuse is an optional setting that enables cookie reuse for
// all requests defined in raw section.
diff --git a/pkg/protocols/http/request.go b/pkg/protocols/http/request.go
index 2eec929808..98e1932477 100644
--- a/pkg/protocols/http/request.go
+++ b/pkg/protocols/http/request.go
@@ -689,7 +689,7 @@ func (request *Request) executeRequest(input *contextargs.Context, generatedRequ
}
// === apply auth strategies ===
- if generatedRequest.request != nil {
+ if generatedRequest.request != nil && !request.SkipSecretFile {
generatedRequest.ApplyAuth(request.options.AuthProvider)
}
From d20ec34f636a827838357b769270e12e8013a872 Mon Sep 17 00:00:00 2001
From: ghost
Date: Fri, 16 Aug 2024 12:41:50 +0000
Subject: [PATCH 144/283] Auto Generate Syntax Docs + JSONSchema [Fri Aug 16
12:41:50 UTC 2024] :robot:
---
SYNTAX-REFERENCE.md | 13 ++++
nuclei-jsonschema.json | 5 ++
pkg/templates/templates_doc.go | 117 +++++++++++++++++----------------
3 files changed, 79 insertions(+), 56 deletions(-)
diff --git a/SYNTAX-REFERENCE.md b/SYNTAX-REFERENCE.md
index d6449e29e4..28bb5bdf24 100755
--- a/SYNTAX-REFERENCE.md
+++ b/SYNTAX-REFERENCE.md
@@ -1404,6 +1404,19 @@ Valid values:
+skip-secret-file
bool
+
+
+
+
+SkipSecretFile skips the authentication or authorization configured in the secret file.
+
+
+
+
+
+
+
cookie-reuse
bool
diff --git a/nuclei-jsonschema.json b/nuclei-jsonschema.json
index 73ba92381e..cd8f8fa0b0 100644
--- a/nuclei-jsonschema.json
+++ b/nuclei-jsonschema.json
@@ -787,6 +787,11 @@
"title": "signature is the http request signature method",
"description": "Signature is the HTTP Request signature Method"
},
+ "skip-secret-file": {
+ "type": "boolean",
+ "title": "bypass secret file",
+ "description": "Skips the authentication or authorization configured in the secret file"
+ },
"cookie-reuse": {
"type": "boolean",
"title": "optional cookie reuse enable",
diff --git a/pkg/templates/templates_doc.go b/pkg/templates/templates_doc.go
index 53864516a7..5171e2a88f 100644
--- a/pkg/templates/templates_doc.go
+++ b/pkg/templates/templates_doc.go
@@ -459,7 +459,7 @@ func init() {
Value: "HTTP response headers in name:value format",
},
}
- HTTPRequestDoc.Fields = make([]encoder.Doc, 35)
+ HTTPRequestDoc.Fields = make([]encoder.Doc, 36)
HTTPRequestDoc.Fields[0].Name = "path"
HTTPRequestDoc.Fields[0].Type = "[]string"
HTTPRequestDoc.Fields[0].Note = ""
@@ -578,91 +578,96 @@ func init() {
HTTPRequestDoc.Fields[17].Values = []string{
"AWS",
}
- HTTPRequestDoc.Fields[18].Name = "cookie-reuse"
+ HTTPRequestDoc.Fields[18].Name = "skip-secret-file"
HTTPRequestDoc.Fields[18].Type = "bool"
HTTPRequestDoc.Fields[18].Note = ""
- HTTPRequestDoc.Fields[18].Description = "CookieReuse is an optional setting that enables cookie reuse for\nall requests defined in raw section."
- HTTPRequestDoc.Fields[18].Comments[encoder.LineComment] = "CookieReuse is an optional setting that enables cookie reuse for"
- HTTPRequestDoc.Fields[19].Name = "disable-cookie"
+ HTTPRequestDoc.Fields[18].Description = "SkipSecretFile skips the authentication or authorization configured in the secret file."
+ HTTPRequestDoc.Fields[18].Comments[encoder.LineComment] = "SkipSecretFile skips the authentication or authorization configured in the secret file."
+ HTTPRequestDoc.Fields[19].Name = "cookie-reuse"
HTTPRequestDoc.Fields[19].Type = "bool"
HTTPRequestDoc.Fields[19].Note = ""
- HTTPRequestDoc.Fields[19].Description = "DisableCookie is an optional setting that disables cookie reuse"
- HTTPRequestDoc.Fields[19].Comments[encoder.LineComment] = "DisableCookie is an optional setting that disables cookie reuse"
- HTTPRequestDoc.Fields[20].Name = "read-all"
+ HTTPRequestDoc.Fields[19].Description = "CookieReuse is an optional setting that enables cookie reuse for\nall requests defined in raw section."
+ HTTPRequestDoc.Fields[19].Comments[encoder.LineComment] = "CookieReuse is an optional setting that enables cookie reuse for"
+ HTTPRequestDoc.Fields[20].Name = "disable-cookie"
HTTPRequestDoc.Fields[20].Type = "bool"
HTTPRequestDoc.Fields[20].Note = ""
- HTTPRequestDoc.Fields[20].Description = "Enables force reading of the entire raw unsafe request body ignoring\nany specified content length headers."
- HTTPRequestDoc.Fields[20].Comments[encoder.LineComment] = "Enables force reading of the entire raw unsafe request body ignoring"
- HTTPRequestDoc.Fields[21].Name = "redirects"
+ HTTPRequestDoc.Fields[20].Description = "DisableCookie is an optional setting that disables cookie reuse"
+ HTTPRequestDoc.Fields[20].Comments[encoder.LineComment] = "DisableCookie is an optional setting that disables cookie reuse"
+ HTTPRequestDoc.Fields[21].Name = "read-all"
HTTPRequestDoc.Fields[21].Type = "bool"
HTTPRequestDoc.Fields[21].Note = ""
- HTTPRequestDoc.Fields[21].Description = "Redirects specifies whether redirects should be followed by the HTTP Client.\n\nThis can be used in conjunction with `max-redirects` to control the HTTP request redirects."
- HTTPRequestDoc.Fields[21].Comments[encoder.LineComment] = "Redirects specifies whether redirects should be followed by the HTTP Client."
- HTTPRequestDoc.Fields[22].Name = "host-redirects"
+ HTTPRequestDoc.Fields[21].Description = "Enables force reading of the entire raw unsafe request body ignoring\nany specified content length headers."
+ HTTPRequestDoc.Fields[21].Comments[encoder.LineComment] = "Enables force reading of the entire raw unsafe request body ignoring"
+ HTTPRequestDoc.Fields[22].Name = "redirects"
HTTPRequestDoc.Fields[22].Type = "bool"
HTTPRequestDoc.Fields[22].Note = ""
- HTTPRequestDoc.Fields[22].Description = "Redirects specifies whether only redirects to the same host should be followed by the HTTP Client.\n\nThis can be used in conjunction with `max-redirects` to control the HTTP request redirects."
- HTTPRequestDoc.Fields[22].Comments[encoder.LineComment] = "Redirects specifies whether only redirects to the same host should be followed by the HTTP Client."
- HTTPRequestDoc.Fields[23].Name = "pipeline"
+ HTTPRequestDoc.Fields[22].Description = "Redirects specifies whether redirects should be followed by the HTTP Client.\n\nThis can be used in conjunction with `max-redirects` to control the HTTP request redirects."
+ HTTPRequestDoc.Fields[22].Comments[encoder.LineComment] = "Redirects specifies whether redirects should be followed by the HTTP Client."
+ HTTPRequestDoc.Fields[23].Name = "host-redirects"
HTTPRequestDoc.Fields[23].Type = "bool"
HTTPRequestDoc.Fields[23].Note = ""
- HTTPRequestDoc.Fields[23].Description = "Pipeline defines if the attack should be performed with HTTP 1.1 Pipelining\n\nAll requests must be idempotent (GET/POST). This can be used for race conditions/billions requests."
- HTTPRequestDoc.Fields[23].Comments[encoder.LineComment] = "Pipeline defines if the attack should be performed with HTTP 1.1 Pipelining"
- HTTPRequestDoc.Fields[24].Name = "unsafe"
+ HTTPRequestDoc.Fields[23].Description = "Redirects specifies whether only redirects to the same host should be followed by the HTTP Client.\n\nThis can be used in conjunction with `max-redirects` to control the HTTP request redirects."
+ HTTPRequestDoc.Fields[23].Comments[encoder.LineComment] = "Redirects specifies whether only redirects to the same host should be followed by the HTTP Client."
+ HTTPRequestDoc.Fields[24].Name = "pipeline"
HTTPRequestDoc.Fields[24].Type = "bool"
HTTPRequestDoc.Fields[24].Note = ""
- HTTPRequestDoc.Fields[24].Description = "Unsafe specifies whether to use rawhttp engine for sending Non RFC-Compliant requests.\n\nThis uses the [rawhttp](https://github.com/projectdiscovery/rawhttp) engine to achieve complete\ncontrol over the request, with no normalization performed by the client."
- HTTPRequestDoc.Fields[24].Comments[encoder.LineComment] = "Unsafe specifies whether to use rawhttp engine for sending Non RFC-Compliant requests."
- HTTPRequestDoc.Fields[25].Name = "race"
+ HTTPRequestDoc.Fields[24].Description = "Pipeline defines if the attack should be performed with HTTP 1.1 Pipelining\n\nAll requests must be idempotent (GET/POST). This can be used for race conditions/billions requests."
+ HTTPRequestDoc.Fields[24].Comments[encoder.LineComment] = "Pipeline defines if the attack should be performed with HTTP 1.1 Pipelining"
+ HTTPRequestDoc.Fields[25].Name = "unsafe"
HTTPRequestDoc.Fields[25].Type = "bool"
HTTPRequestDoc.Fields[25].Note = ""
- HTTPRequestDoc.Fields[25].Description = "Race determines if all the request have to be attempted at the same time (Race Condition)\n\nThe actual number of requests that will be sent is determined by the `race_count` field."
- HTTPRequestDoc.Fields[25].Comments[encoder.LineComment] = "Race determines if all the request have to be attempted at the same time (Race Condition)"
- HTTPRequestDoc.Fields[26].Name = "req-condition"
+ HTTPRequestDoc.Fields[25].Description = "Unsafe specifies whether to use rawhttp engine for sending Non RFC-Compliant requests.\n\nThis uses the [rawhttp](https://github.com/projectdiscovery/rawhttp) engine to achieve complete\ncontrol over the request, with no normalization performed by the client."
+ HTTPRequestDoc.Fields[25].Comments[encoder.LineComment] = "Unsafe specifies whether to use rawhttp engine for sending Non RFC-Compliant requests."
+ HTTPRequestDoc.Fields[26].Name = "race"
HTTPRequestDoc.Fields[26].Type = "bool"
HTTPRequestDoc.Fields[26].Note = ""
- HTTPRequestDoc.Fields[26].Description = "ReqCondition automatically assigns numbers to requests and preserves their history.\n\nThis allows matching on them later for multi-request conditions."
- HTTPRequestDoc.Fields[26].Comments[encoder.LineComment] = "ReqCondition automatically assigns numbers to requests and preserves their history."
- HTTPRequestDoc.Fields[27].Name = "stop-at-first-match"
+ HTTPRequestDoc.Fields[26].Description = "Race determines if all the request have to be attempted at the same time (Race Condition)\n\nThe actual number of requests that will be sent is determined by the `race_count` field."
+ HTTPRequestDoc.Fields[26].Comments[encoder.LineComment] = "Race determines if all the request have to be attempted at the same time (Race Condition)"
+ HTTPRequestDoc.Fields[27].Name = "req-condition"
HTTPRequestDoc.Fields[27].Type = "bool"
HTTPRequestDoc.Fields[27].Note = ""
- HTTPRequestDoc.Fields[27].Description = "StopAtFirstMatch stops the execution of the requests and template as soon as a match is found."
- HTTPRequestDoc.Fields[27].Comments[encoder.LineComment] = "StopAtFirstMatch stops the execution of the requests and template as soon as a match is found."
- HTTPRequestDoc.Fields[28].Name = "skip-variables-check"
+ HTTPRequestDoc.Fields[27].Description = "ReqCondition automatically assigns numbers to requests and preserves their history.\n\nThis allows matching on them later for multi-request conditions."
+ HTTPRequestDoc.Fields[27].Comments[encoder.LineComment] = "ReqCondition automatically assigns numbers to requests and preserves their history."
+ HTTPRequestDoc.Fields[28].Name = "stop-at-first-match"
HTTPRequestDoc.Fields[28].Type = "bool"
HTTPRequestDoc.Fields[28].Note = ""
- HTTPRequestDoc.Fields[28].Description = "SkipVariablesCheck skips the check for unresolved variables in request"
- HTTPRequestDoc.Fields[28].Comments[encoder.LineComment] = "SkipVariablesCheck skips the check for unresolved variables in request"
- HTTPRequestDoc.Fields[29].Name = "iterate-all"
+ HTTPRequestDoc.Fields[28].Description = "StopAtFirstMatch stops the execution of the requests and template as soon as a match is found."
+ HTTPRequestDoc.Fields[28].Comments[encoder.LineComment] = "StopAtFirstMatch stops the execution of the requests and template as soon as a match is found."
+ HTTPRequestDoc.Fields[29].Name = "skip-variables-check"
HTTPRequestDoc.Fields[29].Type = "bool"
HTTPRequestDoc.Fields[29].Note = ""
- HTTPRequestDoc.Fields[29].Description = "IterateAll iterates all the values extracted from internal extractors"
- HTTPRequestDoc.Fields[29].Comments[encoder.LineComment] = "IterateAll iterates all the values extracted from internal extractors"
- HTTPRequestDoc.Fields[30].Name = "digest-username"
- HTTPRequestDoc.Fields[30].Type = "string"
+ HTTPRequestDoc.Fields[29].Description = "SkipVariablesCheck skips the check for unresolved variables in request"
+ HTTPRequestDoc.Fields[29].Comments[encoder.LineComment] = "SkipVariablesCheck skips the check for unresolved variables in request"
+ HTTPRequestDoc.Fields[30].Name = "iterate-all"
+ HTTPRequestDoc.Fields[30].Type = "bool"
HTTPRequestDoc.Fields[30].Note = ""
- HTTPRequestDoc.Fields[30].Description = "DigestAuthUsername specifies the username for digest authentication"
- HTTPRequestDoc.Fields[30].Comments[encoder.LineComment] = "DigestAuthUsername specifies the username for digest authentication"
- HTTPRequestDoc.Fields[31].Name = "digest-password"
+ HTTPRequestDoc.Fields[30].Description = "IterateAll iterates all the values extracted from internal extractors"
+ HTTPRequestDoc.Fields[30].Comments[encoder.LineComment] = "IterateAll iterates all the values extracted from internal extractors"
+ HTTPRequestDoc.Fields[31].Name = "digest-username"
HTTPRequestDoc.Fields[31].Type = "string"
HTTPRequestDoc.Fields[31].Note = ""
- HTTPRequestDoc.Fields[31].Description = "DigestAuthPassword specifies the password for digest authentication"
- HTTPRequestDoc.Fields[31].Comments[encoder.LineComment] = "DigestAuthPassword specifies the password for digest authentication"
- HTTPRequestDoc.Fields[32].Name = "disable-path-automerge"
- HTTPRequestDoc.Fields[32].Type = "bool"
+ HTTPRequestDoc.Fields[31].Description = "DigestAuthUsername specifies the username for digest authentication"
+ HTTPRequestDoc.Fields[31].Comments[encoder.LineComment] = "DigestAuthUsername specifies the username for digest authentication"
+ HTTPRequestDoc.Fields[32].Name = "digest-password"
+ HTTPRequestDoc.Fields[32].Type = "string"
HTTPRequestDoc.Fields[32].Note = ""
- HTTPRequestDoc.Fields[32].Description = "DisablePathAutomerge disables merging target url path with raw request path"
- HTTPRequestDoc.Fields[32].Comments[encoder.LineComment] = "DisablePathAutomerge disables merging target url path with raw request path"
- HTTPRequestDoc.Fields[33].Name = "pre-condition"
- HTTPRequestDoc.Fields[33].Type = "[]matchers.Matcher"
+ HTTPRequestDoc.Fields[32].Description = "DigestAuthPassword specifies the password for digest authentication"
+ HTTPRequestDoc.Fields[32].Comments[encoder.LineComment] = "DigestAuthPassword specifies the password for digest authentication"
+ HTTPRequestDoc.Fields[33].Name = "disable-path-automerge"
+ HTTPRequestDoc.Fields[33].Type = "bool"
HTTPRequestDoc.Fields[33].Note = ""
- HTTPRequestDoc.Fields[33].Description = "Fuzz PreCondition is matcher-like field to check if fuzzing should be performed on this request or not"
- HTTPRequestDoc.Fields[33].Comments[encoder.LineComment] = "Fuzz PreCondition is matcher-like field to check if fuzzing should be performed on this request or not"
- HTTPRequestDoc.Fields[34].Name = "pre-condition-operator"
- HTTPRequestDoc.Fields[34].Type = "string"
+ HTTPRequestDoc.Fields[33].Description = "DisablePathAutomerge disables merging target url path with raw request path"
+ HTTPRequestDoc.Fields[33].Comments[encoder.LineComment] = "DisablePathAutomerge disables merging target url path with raw request path"
+ HTTPRequestDoc.Fields[34].Name = "pre-condition"
+ HTTPRequestDoc.Fields[34].Type = "[]matchers.Matcher"
HTTPRequestDoc.Fields[34].Note = ""
- HTTPRequestDoc.Fields[34].Description = "FuzzPreConditionOperator is the operator between multiple PreConditions for fuzzing Default is OR"
- HTTPRequestDoc.Fields[34].Comments[encoder.LineComment] = "FuzzPreConditionOperator is the operator between multiple PreConditions for fuzzing Default is OR"
+ HTTPRequestDoc.Fields[34].Description = "Fuzz PreCondition is matcher-like field to check if fuzzing should be performed on this request or not"
+ HTTPRequestDoc.Fields[34].Comments[encoder.LineComment] = "Fuzz PreCondition is matcher-like field to check if fuzzing should be performed on this request or not"
+ HTTPRequestDoc.Fields[35].Name = "pre-condition-operator"
+ HTTPRequestDoc.Fields[35].Type = "string"
+ HTTPRequestDoc.Fields[35].Note = ""
+ HTTPRequestDoc.Fields[35].Description = "FuzzPreConditionOperator is the operator between multiple PreConditions for fuzzing Default is OR"
+ HTTPRequestDoc.Fields[35].Comments[encoder.LineComment] = "FuzzPreConditionOperator is the operator between multiple PreConditions for fuzzing Default is OR"
GENERATORSAttackTypeHolderDoc.Type = "generators.AttackTypeHolder"
GENERATORSAttackTypeHolderDoc.Comments[encoder.LineComment] = " AttackTypeHolder is used to hold internal type of the protocol"
From f29b94521e4882084ddcbd8fc7c61d75b7aaa37c Mon Sep 17 00:00:00 2001
From: Ramana Reddy <90540245+RamanaReddy0M@users.noreply.github.com>
Date: Fri, 16 Aug 2024 18:19:44 +0530
Subject: [PATCH 145/283] fix unresolved variables in dast templates (#5443)
* fix unresolved variables in dast templates
* dedupe interactsh urls
* misc update
---
pkg/fuzz/execute.go | 13 ++++++++++++-
pkg/protocols/http/request_fuzz.go | 14 ++++++++------
2 files changed, 20 insertions(+), 7 deletions(-)
diff --git a/pkg/fuzz/execute.go b/pkg/fuzz/execute.go
index 8eb57f6b40..2591ec39ac 100644
--- a/pkg/fuzz/execute.go
+++ b/pkg/fuzz/execute.go
@@ -16,6 +16,7 @@ import (
"github.com/projectdiscovery/nuclei/v3/pkg/protocols/common/generators"
"github.com/projectdiscovery/retryablehttp-go"
errorutil "github.com/projectdiscovery/utils/errors"
+ mapsutil "github.com/projectdiscovery/utils/maps"
sliceutil "github.com/projectdiscovery/utils/slice"
urlutil "github.com/projectdiscovery/utils/url"
)
@@ -165,6 +166,11 @@ mainLoop:
func (rule *Rule) evaluateVarsWithInteractsh(data map[string]interface{}, interactshUrls []string) (map[string]interface{}, []string) {
// Check if Interactsh options are configured
if rule.options.Interactsh != nil {
+ interactshUrlsMap := make(map[string]struct{})
+ for _, url := range interactshUrls {
+ interactshUrlsMap[url] = struct{}{}
+ }
+ interactshUrls = mapsutil.GetKeys(interactshUrlsMap)
// Iterate through the data to replace and evaluate variables with Interactsh URLs
for k, v := range data {
value := fmt.Sprint(v)
@@ -175,7 +181,12 @@ func (rule *Rule) evaluateVarsWithInteractsh(data map[string]interface{}, intera
}
// Append new OAST URLs if any
if len(oastUrls) > 0 {
- interactshUrls = append(interactshUrls, oastUrls...)
+ for _, url := range oastUrls {
+ if _, ok := interactshUrlsMap[url]; !ok {
+ interactshUrlsMap[url] = struct{}{}
+ interactshUrls = append(interactshUrls, url)
+ }
+ }
}
// Evaluate the replaced data
evaluatedData, err := expressions.Evaluate(got, data)
diff --git a/pkg/protocols/http/request_fuzz.go b/pkg/protocols/http/request_fuzz.go
index 49df3c0632..fdf862eb67 100644
--- a/pkg/protocols/http/request_fuzz.go
+++ b/pkg/protocols/http/request_fuzz.go
@@ -18,6 +18,7 @@ import (
"github.com/projectdiscovery/nuclei/v3/pkg/output"
"github.com/projectdiscovery/nuclei/v3/pkg/protocols"
"github.com/projectdiscovery/nuclei/v3/pkg/protocols/common/contextargs"
+ "github.com/projectdiscovery/nuclei/v3/pkg/protocols/common/generators"
"github.com/projectdiscovery/nuclei/v3/pkg/protocols/common/interactsh"
"github.com/projectdiscovery/nuclei/v3/pkg/protocols/common/utils/vardump"
protocolutils "github.com/projectdiscovery/nuclei/v3/pkg/protocols/utils"
@@ -112,6 +113,7 @@ func (request *Request) executeFuzzingRule(input *contextargs.Context, previous
// executeAllFuzzingRules executes all fuzzing rules defined in template for a given base request
func (request *Request) executeAllFuzzingRules(input *contextargs.Context, values map[string]interface{}, baseRequest *retryablehttp.Request, callback protocols.OutputEventCallback) error {
applicable := false
+ values = generators.MergeMaps(request.filterDataMap(input), values)
for _, rule := range request.Fuzzing {
select {
case <-input.Context().Done():
@@ -234,7 +236,12 @@ func (request *Request) ShouldFuzzTarget(input *contextargs.Context) bool {
}
status := []bool{}
for index, filter := range request.FuzzPreCondition {
- isMatch, _ := request.Match(request.filterDataMap(input), filter)
+ dataMap := request.filterDataMap(input)
+ // dump if svd is enabled
+ if request.options.Options.ShowVarDump {
+ gologger.Debug().Msgf("Fuzz Filter Variables: \n%s\n", vardump.DumpVariables(dataMap))
+ }
+ isMatch, _ := request.Match(dataMap, filter)
status = append(status, isMatch)
if request.options.Options.MatcherStatus {
gologger.Debug().Msgf("[%s] [%s] Filter => %s : %v", input.MetaInput.Target(), request.options.TemplateID, operators.GetMatcherName(filter, index), isMatch)
@@ -295,10 +302,5 @@ func (request *Request) filterDataMap(input *contextargs.Context) map[string]int
// add default method value
m["method"] = http.MethodGet
}
-
- // dump if svd is enabled
- if request.options.Options.ShowVarDump {
- gologger.Debug().Msgf("Fuzz Filter Variables: \n%s\n", vardump.DumpVariables(m))
- }
return m
}
From 0675aa48a348d14458329edfc14f16cac7cfc5b1 Mon Sep 17 00:00:00 2001
From: Tryfon Papatriantafyllou <67585616+trypa11@users.noreply.github.com>
Date: Fri, 16 Aug 2024 16:04:47 +0300
Subject: [PATCH 146/283] Circular References in OpenAPI 3.0 fixed (#5491)
* Circular References in OpenAPI 3.0 fixed
* Fixing Swagger_test
---
go.mod | 8 ++++----
go.sum | 17 ++++++++---------
pkg/input/formats/openapi/examples.go | 12 ++++++------
pkg/input/formats/swagger/swagger.go | 9 +++++++--
4 files changed, 25 insertions(+), 21 deletions(-)
diff --git a/go.mod b/go.mod
index 0f729d5598..f2053614c1 100644
--- a/go.mod
+++ b/go.mod
@@ -65,12 +65,13 @@ require (
github.com/docker/go-units v0.5.0
github.com/dop251/goja v0.0.0-20240220182346-e401ed450204
github.com/fatih/structs v1.1.0
- github.com/getkin/kin-openapi v0.123.0
+ github.com/getkin/kin-openapi v0.126.0
github.com/go-git/go-git/v5 v5.11.0
github.com/go-ldap/ldap/v3 v3.4.5
github.com/go-pg/pg v8.0.7+incompatible
github.com/go-sql-driver/mysql v1.7.1
github.com/h2non/filetype v1.1.3
+ github.com/invopop/yaml v0.3.1
github.com/labstack/echo/v4 v4.10.2
github.com/leslie-qiwa/flat v0.0.0-20230424180412-f9d1cf014baa
github.com/lib/pq v1.10.9
@@ -152,8 +153,8 @@ require (
github.com/gin-gonic/gin v1.9.1 // indirect
github.com/go-asn1-ber/asn1-ber v1.5.4 // indirect
github.com/go-fed/httpsig v1.1.0 // indirect
- github.com/go-openapi/jsonpointer v0.20.2 // indirect
- github.com/go-openapi/swag v0.22.9 // indirect
+ github.com/go-openapi/jsonpointer v0.21.0 // indirect
+ github.com/go-openapi/swag v0.23.0 // indirect
github.com/go-sourcemap/sourcemap v2.1.4+incompatible // indirect
github.com/goccy/go-json v0.10.2 // indirect
github.com/gogo/protobuf v1.3.2 // indirect
@@ -168,7 +169,6 @@ require (
github.com/hashicorp/go-version v1.6.0 // indirect
github.com/hashicorp/golang-lru/v2 v2.0.6 // indirect
github.com/hbakhtiyor/strsim v0.0.0-20190107154042-4d2bbb273edf // indirect
- github.com/invopop/yaml v0.2.0 // indirect
github.com/jcmturner/aescts/v2 v2.0.0 // indirect
github.com/jcmturner/dnsutils/v2 v2.0.0 // indirect
github.com/jcmturner/gofork v1.7.6 // indirect
diff --git a/go.sum b/go.sum
index 3f8cef8e83..dadb721a3f 100644
--- a/go.sum
+++ b/go.sum
@@ -329,8 +329,8 @@ github.com/gaissmai/bart v0.9.5 h1:vy+r4Px6bjZ+v2QYXAsg63vpz9IfzdW146A8Cn4GPIo=
github.com/gaissmai/bart v0.9.5/go.mod h1:KHeYECXQiBjTzQz/om2tqn3sZF1J7hw9m6z41ftj3fg=
github.com/geoffgarside/ber v1.1.0 h1:qTmFG4jJbwiSzSXoNJeHcOprVzZ8Ulde2Rrrifu5U9w=
github.com/geoffgarside/ber v1.1.0/go.mod h1:jVPKeCbj6MvQZhwLYsGwaGI52oUorHoHKNecGT85ZCc=
-github.com/getkin/kin-openapi v0.123.0 h1:zIik0mRwFNLyvtXK274Q6ut+dPh6nlxBp0x7mNrPhs8=
-github.com/getkin/kin-openapi v0.123.0/go.mod h1:wb1aSZA/iWmorQP9KTAS/phLj/t17B5jT7+fS8ed9NM=
+github.com/getkin/kin-openapi v0.126.0 h1:c2cSgLnAsS0xYfKsgt5oBV6MYRM/giU8/RtwUY4wyfY=
+github.com/getkin/kin-openapi v0.126.0/go.mod h1:7mONz8IwmSRg6RttPu6v8U/OJ+gr+J99qSFNjPGSQqw=
github.com/ghodss/yaml v1.0.0/go.mod h1:4dBDuWmgqj2HViK6kFavaiC9ZROes6MMH2rRYeMEF04=
github.com/gin-contrib/sse v0.1.0 h1:Y/yl/+YNO8GZSjAhjMsSuLt29uWRFHdHYUb5lYOV9qE=
github.com/gin-contrib/sse v0.1.0/go.mod h1:RHrZQHXnP2xjPF+u1gW/2HnVO7nvIa9PG3Gm+fLHvGI=
@@ -369,10 +369,10 @@ github.com/go-logfmt/logfmt v0.5.0/go.mod h1:wCYkCAKZfumFQihp8CzCvQ3paCTfi41vtzG
github.com/go-logfmt/logfmt v0.5.1/go.mod h1:WYhtIu8zTZfxdn5+rREduYbwxfcBr/Vr6KEVveWlfTs=
github.com/go-ole/go-ole v1.2.6 h1:/Fpf6oFPoeFik9ty7siob0G6Ke8QvQEuVcuChpwXzpY=
github.com/go-ole/go-ole v1.2.6/go.mod h1:pprOEPIfldk/42T2oK7lQ4v4JSDwmV0As9GaiUsvbm0=
-github.com/go-openapi/jsonpointer v0.20.2 h1:mQc3nmndL8ZBzStEo3JYF8wzmeWffDH4VbXz58sAx6Q=
-github.com/go-openapi/jsonpointer v0.20.2/go.mod h1:bHen+N0u1KEO3YlmqOjTT9Adn1RfD91Ar825/PuiRVs=
-github.com/go-openapi/swag v0.22.9 h1:XX2DssF+mQKM2DHsbgZK74y/zj4mo9I99+89xUmuZCE=
-github.com/go-openapi/swag v0.22.9/go.mod h1:3/OXnFfnMAwBD099SwYRk7GD3xOrr1iL7d/XNLXVVwE=
+github.com/go-openapi/jsonpointer v0.21.0 h1:YgdVicSA9vH5RiHs9TZW5oyafXZFc6+2Vc1rr/O9oNQ=
+github.com/go-openapi/jsonpointer v0.21.0/go.mod h1:IUyH9l/+uyhIYQ/PXVA41Rexl+kOkAPDdXEYns6fzUY=
+github.com/go-openapi/swag v0.23.0 h1:vsEVJDUo2hPJ2tu0/Xc+4noaxyEffXNIs3cOULZ+GrE=
+github.com/go-openapi/swag v0.23.0/go.mod h1:esZ8ITTYEsH1V2trKHjAN8Ai7xHb8RV+YSZ577vPjgQ=
github.com/go-pg/pg v8.0.7+incompatible h1:ty/sXL1OZLo+47KK9N8llRcmbA9tZasqbQ/OO4ld53g=
github.com/go-pg/pg v8.0.7+incompatible/go.mod h1:a2oXow+aFOrvwcKs3eIA0lNFmMilrxK2sOkB5NWe0vA=
github.com/go-playground/assert/v2 v2.2.0 h1:JvknZsQTYeFEAhQwI4qEt9cyV5ONwRHC+lYKSsYSR8s=
@@ -572,8 +572,8 @@ github.com/inconshreveable/mousetrap v1.0.0/go.mod h1:PxqpIevigyE2G7u3NXJIT2ANyt
github.com/influxdata/influxdb1-client v0.0.0-20191209144304-8bf82d3c094d/go.mod h1:qj24IKcXYK6Iy9ceXlo3Tc+vtHo9lIhSX5JddghvEPo=
github.com/invopop/jsonschema v0.12.0 h1:6ovsNSuvn9wEQVOyc72aycBMVQFKz7cPdMJn10CvzRI=
github.com/invopop/jsonschema v0.12.0/go.mod h1:ffZ5Km5SWWRAIN6wbDXItl95euhFz2uON45H2qjYt+0=
-github.com/invopop/yaml v0.2.0 h1:7zky/qH+O0DwAyoobXUqvVBwgBFRxKoQ/3FjcVpjTMY=
-github.com/invopop/yaml v0.2.0/go.mod h1:2XuRLgs/ouIrW3XNzuNj7J3Nvu/Dig5MXvbCEdiBN3Q=
+github.com/invopop/yaml v0.3.1 h1:f0+ZpmhfBSS4MhG+4HYseMdJhoeeopbSKbq5Rpeelso=
+github.com/invopop/yaml v0.3.1/go.mod h1:PMOp3nn4/12yEZUFfmOuNHJsZToEEOwoWsT+D81KkeA=
github.com/itchyny/gojq v0.12.13 h1:IxyYlHYIlspQHHTE0f3cJF0NKDMfajxViuhBLnHd/QU=
github.com/itchyny/gojq v0.12.13/go.mod h1:JzwzAqenfhrPUuwbmEz3nu3JQmFLlQTQMUcOdnu/Sf4=
github.com/itchyny/timefmt-go v0.1.5 h1:G0INE2la8S6ru/ZI5JecgyzbbJNs5lG1RcBqa7Jm6GE=
@@ -1604,7 +1604,6 @@ gopkg.in/yaml.v2 v2.4.0 h1:D8xgwECY7CYvx+Y2n4sBz93Jn9JRvxdiyyo8CTfuKaY=
gopkg.in/yaml.v2 v2.4.0/go.mod h1:RDklbk79AGWmwhnvt/jBztapEOGDOx6ZbXqjP6csGnQ=
gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
-gopkg.in/yaml.v3 v3.0.0/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA=
gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
gotest.tools/v3 v3.3.0 h1:MfDY1b1/0xN1CyMlQDac0ziEy9zJQd9CXBRRDHw2jJo=
diff --git a/pkg/input/formats/openapi/examples.go b/pkg/input/formats/openapi/examples.go
index 35c4292cb1..3b788ca42b 100644
--- a/pkg/input/formats/openapi/examples.go
+++ b/pkg/input/formats/openapi/examples.go
@@ -175,9 +175,9 @@ func openAPIExample(schema *openapi3.Schema, cache map[*openapi3.Schema]*cachedS
}
switch {
- case schema.Type == "boolean":
+ case schema.Type.Is("boolean"):
return true, nil
- case schema.Type == "number", schema.Type == "integer":
+ case schema.Type.Is("number"), schema.Type.Is("integer"):
value := 0.0
if schema.Min != nil && *schema.Min > value {
@@ -208,11 +208,11 @@ func openAPIExample(schema *openapi3.Schema, cache map[*openapi3.Schema]*cachedS
value += float64(int(*schema.MultipleOf) - (int(value) % int(*schema.MultipleOf)))
}
- if schema.Type == "integer" {
+ if schema.Type.Is("integer") {
return int(value), nil
}
return value, nil
- case schema.Type == "string":
+ case schema.Type.Is("string"):
if ex := stringFormatExample(schema.Format); ex != "" {
return ex, nil
}
@@ -226,7 +226,7 @@ func openAPIExample(schema *openapi3.Schema, cache map[*openapi3.Schema]*cachedS
example = example[:*schema.MaxLength]
}
return example, nil
- case schema.Type == "array", schema.Items != nil:
+ case schema.Type.Is("array"), schema.Items != nil:
example := []interface{}{}
if schema.Items != nil && schema.Items.Value != nil {
@@ -242,7 +242,7 @@ func openAPIExample(schema *openapi3.Schema, cache map[*openapi3.Schema]*cachedS
}
}
return example, nil
- case schema.Type == "object", len(schema.Properties) > 0:
+ case schema.Type.Is("object"), len(schema.Properties) > 0:
example := map[string]interface{}{}
for k, v := range schema.Properties {
diff --git a/pkg/input/formats/swagger/swagger.go b/pkg/input/formats/swagger/swagger.go
index 2828bb293e..30a7564ecc 100644
--- a/pkg/input/formats/swagger/swagger.go
+++ b/pkg/input/formats/swagger/swagger.go
@@ -2,6 +2,7 @@ package swagger
import (
"encoding/json"
+ "io"
"os"
"path"
@@ -10,7 +11,7 @@ import (
"github.com/pkg/errors"
"github.com/projectdiscovery/nuclei/v3/pkg/input/formats"
"github.com/projectdiscovery/nuclei/v3/pkg/input/formats/openapi"
- "gopkg.in/yaml.v2"
+ "github.com/invopop/yaml"
"github.com/getkin/kin-openapi/openapi2conv"
)
@@ -49,7 +50,11 @@ func (j *SwaggerFormat) Parse(input string, resultsCb formats.ParseReqRespCallba
ext := path.Ext(input)
if ext == ".yaml" || ext == ".yml" {
- err = yaml.NewDecoder(file).Decode(schemav2)
+ data, err_data := io.ReadAll(file)
+ if err_data != nil {
+ return errors.Wrap(err, "could not read data file")
+ }
+ err = yaml.Unmarshal(data, schemav2)
} else {
err = json.NewDecoder(file).Decode(schemav2)
}
From 1c76398aeac2b35be6decb197f6303055e9e06a5 Mon Sep 17 00:00:00 2001
From: Tarun Koyalwar <45962551+tarunKoyalwar@users.noreply.github.com>
Date: Fri, 16 Aug 2024 08:01:23 -0700
Subject: [PATCH 147/283] lint error fixes (#5531)
* lint error fixes
* chore: satisfy non-constant format str in call lint (govet)
Signed-off-by: Dwi Siswanto
---------
Signed-off-by: Dwi Siswanto
Co-authored-by: Dwi Siswanto
---
cmd/tmc/main.go | 8 ++++----
lib/example_test.go | 2 +-
pkg/input/formats/openapi/examples.go | 2 +-
pkg/input/formats/openapi/generator.go | 4 ++--
pkg/js/devtools/bindgen/generator.go | 2 +-
pkg/js/devtools/tsgen/parser.go | 2 +-
pkg/js/utils/nucleijs.go | 10 +++++++---
pkg/output/output.go | 4 ++--
pkg/projectfile/httputil.go | 2 +-
.../common/helpers/responsehighlighter/hexdump.go | 3 ++-
pkg/protocols/dns/request.go | 2 +-
pkg/protocols/headless/request.go | 2 +-
pkg/reporting/reporting.go | 2 +-
pkg/tmplexec/exec.go | 4 +++-
14 files changed, 28 insertions(+), 21 deletions(-)
diff --git a/cmd/tmc/main.go b/cmd/tmc/main.go
index eae38a1ca7..5cb38afbee 100644
--- a/cmd/tmc/main.go
+++ b/cmd/tmc/main.go
@@ -162,7 +162,7 @@ func process(opts options) error {
var updated bool // if max-requests is updated
dataString, updated, err = parseAndAddMaxRequests(templateCatalog, path, dataString)
if err != nil {
- gologger.Info().Label("max-request").Msgf(logErrMsg(path, err, opts.debug, errFile))
+ gologger.Info().Label("max-request").Msg(logErrMsg(path, err, opts.debug, errFile))
} else {
if updated {
gologger.Info().Label("max-request").Msgf("✅ updated template: %s\n", path)
@@ -255,7 +255,7 @@ func enhanceTemplate(data string) (string, bool, error) {
return data, false, errorutil.New("validation failed").WithTag("validate")
}
if templateResp.Error.Name != "" {
- return data, false, errorutil.New(templateResp.Error.Name)
+ return data, false, errorutil.New("%s", templateResp.Error.Name)
}
if strings.TrimSpace(templateResp.Enhanced) == "" && !templateResp.Lint {
if templateResp.LintError.Reason != "" {
@@ -289,7 +289,7 @@ func formatTemplate(data string) (string, bool, error) {
return data, false, errorutil.New("validation failed").WithTag("validate")
}
if templateResp.Error.Name != "" {
- return data, false, errorutil.New(templateResp.Error.Name)
+ return data, false, errorutil.New("%s", templateResp.Error.Name)
}
if strings.TrimSpace(templateResp.Updated) == "" && !templateResp.Lint {
if templateResp.LintError.Reason != "" {
@@ -345,7 +345,7 @@ func validateTemplate(data string) (bool, error) {
return false, errorutil.New("validation failed").WithTag("validate")
}
if validateResp.Error.Name != "" {
- return false, errorutil.New(validateResp.Error.Name)
+ return false, errorutil.New("%s", validateResp.Error.Name)
}
return false, errorutil.New("template validation failed")
}
diff --git a/lib/example_test.go b/lib/example_test.go
index 1bbc20e584..1c705677a2 100644
--- a/lib/example_test.go
+++ b/lib/example_test.go
@@ -82,5 +82,5 @@ func TestMain(m *testing.M) {
// no need to run this test on github actions
return
}
- m.Run()
+ os.Exit(m.Run())
}
diff --git a/pkg/input/formats/openapi/examples.go b/pkg/input/formats/openapi/examples.go
index 3b788ca42b..9e7224ab73 100644
--- a/pkg/input/formats/openapi/examples.go
+++ b/pkg/input/formats/openapi/examples.go
@@ -18,7 +18,7 @@ func getSchemaExample(schema *openapi3.Schema) (interface{}, bool) {
return schema.Default, true
}
- if schema.Enum != nil && len(schema.Enum) > 0 {
+ if len(schema.Enum) > 0 {
return schema.Enum[0], true
}
return nil, false
diff --git a/pkg/input/formats/openapi/generator.go b/pkg/input/formats/openapi/generator.go
index adcb26b2fa..4027d76fc7 100644
--- a/pkg/input/formats/openapi/generator.go
+++ b/pkg/input/formats/openapi/generator.go
@@ -72,7 +72,7 @@ func GenerateRequestsFromSchema(schema *openapi3.T, opts formats.InputFormatOpti
}
missingVarMap[param.Name] = struct{}{}
}
-
+
for _, serverURL := range schema.Servers {
pathURL := serverURL.URL
// Split the server URL into baseURL and serverPath
@@ -203,7 +203,7 @@ func generateRequestsFromOp(opts *generateReqOptions) error {
paramValue = value.Schema.Value.Default
} else if value.Schema.Value.Example != nil {
paramValue = value.Schema.Value.Example
- } else if value.Schema.Value.Enum != nil && len(value.Schema.Value.Enum) > 0 {
+ } else if len(value.Schema.Value.Enum) > 0 {
paramValue = value.Schema.Value.Enum[0]
} else {
if !opts.opts.SkipFormatValidation {
diff --git a/pkg/js/devtools/bindgen/generator.go b/pkg/js/devtools/bindgen/generator.go
index 313c6d41ca..2a58ca7f0d 100644
--- a/pkg/js/devtools/bindgen/generator.go
+++ b/pkg/js/devtools/bindgen/generator.go
@@ -249,7 +249,7 @@ func identifyGenDecl(pkg *ast.Package, decl *ast.GenDecl, data *TemplateData) {
if !spec.Names[0].IsExported() {
continue
}
- if spec.Values == nil || len(spec.Values) == 0 {
+ if len(spec.Values) == 0 {
continue
}
data.PackageVars[spec.Names[0].Name] = spec.Names[0].Name
diff --git a/pkg/js/devtools/tsgen/parser.go b/pkg/js/devtools/tsgen/parser.go
index e285b5f007..9f7d81ed7b 100644
--- a/pkg/js/devtools/tsgen/parser.go
+++ b/pkg/js/devtools/tsgen/parser.go
@@ -489,7 +489,7 @@ func (p *EntityParser) extractVarsNConstants() {
if !spec.Names[0].IsExported() {
continue
}
- if spec.Values == nil || len(spec.Values) == 0 {
+ if len(spec.Values) == 0 {
continue
}
// get comments or description
diff --git a/pkg/js/utils/nucleijs.go b/pkg/js/utils/nucleijs.go
index d3d4560813..9d9e3f4ece 100644
--- a/pkg/js/utils/nucleijs.go
+++ b/pkg/js/utils/nucleijs.go
@@ -60,12 +60,16 @@ func (j *NucleiJS) HandleError(err error, msg ...string) {
if len(msg) == 0 {
j.ThrowError(err)
}
- j.Throw(fmt.Sprintf("%s: %s", strings.Join(msg, ":"), err.Error()))
+ j.Throw("%s: %s", strings.Join(msg, ":"), err.Error())
}
// Throw throws an error in goja runtime
func (j *NucleiJS) Throw(format string, args ...interface{}) {
- panic(j.runtime().ToValue(fmt.Sprintf(format, args...)))
+ if len(args) > 0 {
+ panic(j.runtime().ToValue(fmt.Sprintf(format, args...)))
+ }
+
+ panic(j.runtime().ToValue(format))
}
// GetArg returns argument at index from goja runtime if not found throws error
@@ -95,7 +99,7 @@ func (j *NucleiJS) GetArgSafe(args []goja.Value, index int, defaultValue any) an
// Require throws an error if expression is false
func (j *NucleiJS) Require(expr bool, msg string) {
if !expr {
- j.Throw(msg)
+ j.Throw("%s", msg)
}
}
diff --git a/pkg/output/output.go b/pkg/output/output.go
index 9f68122b0f..4f02d71c38 100644
--- a/pkg/output/output.go
+++ b/pkg/output/output.go
@@ -525,12 +525,12 @@ func tryParseCause(err error) error {
if strings.HasPrefix(msg, "ReadStatusLine:") {
// last index is actual error (from rawhttp)
parts := strings.Split(msg, ":")
- return errkit.New(strings.TrimSpace(parts[len(parts)-1]))
+ return errkit.New("%s", strings.TrimSpace(parts[len(parts)-1]))
}
if strings.Contains(msg, "read ") {
// same here
parts := strings.Split(msg, ":")
- return errkit.New(strings.TrimSpace(parts[len(parts)-1]))
+ return errkit.New("%s", strings.TrimSpace(parts[len(parts)-1]))
}
return err
}
diff --git a/pkg/projectfile/httputil.go b/pkg/projectfile/httputil.go
index 3f00b8e811..dafeff3fd8 100644
--- a/pkg/projectfile/httputil.go
+++ b/pkg/projectfile/httputil.go
@@ -17,7 +17,7 @@ func hash(v interface{}) (string, error) {
sh := sha256.New()
- if _, err = io.WriteString(sh, string(data)); err != nil {
+ if _, err = sh.Write(data); err != nil {
return "", err
}
return hex.EncodeToString(sh.Sum(nil)), nil
diff --git a/pkg/protocols/common/helpers/responsehighlighter/hexdump.go b/pkg/protocols/common/helpers/responsehighlighter/hexdump.go
index decc985c72..36d60c34da 100644
--- a/pkg/protocols/common/helpers/responsehighlighter/hexdump.go
+++ b/pkg/protocols/common/helpers/responsehighlighter/hexdump.go
@@ -44,7 +44,8 @@ func toHighLightedHexDump(hexDump, snippetToHighlight string) (HighlightableHexD
hexDumpRowValues := hexDumpParsePattern.FindAllStringSubmatch(hexDump, -1)
if hexDumpRowValues == nil || len(hexDumpRowValues) != strings.Count(hexDump, "\n") {
message := "could not parse hexdump"
- gologger.Warning().Msgf(message)
+ gologger.Warning().Msg(message)
+
return HighlightableHexDump{}, errors.New(message)
}
diff --git a/pkg/protocols/dns/request.go b/pkg/protocols/dns/request.go
index 83501c15b1..a16c2af88b 100644
--- a/pkg/protocols/dns/request.go
+++ b/pkg/protocols/dns/request.go
@@ -142,7 +142,7 @@ func (request *Request) execute(input *contextargs.Context, domain string, metad
if request.options.Options.Debug || request.options.Options.DebugRequests || request.options.Options.StoreResponse {
msg := fmt.Sprintf("[%s] Dumped DNS request for %s", request.options.TemplateID, question)
if request.options.Options.Debug || request.options.Options.DebugRequests {
- gologger.Info().Str("domain", domain).Msgf(msg)
+ gologger.Info().Str("domain", domain).Msg(msg)
gologger.Print().Msgf("%s", requestString)
}
if request.options.Options.StoreResponse {
diff --git a/pkg/protocols/headless/request.go b/pkg/protocols/headless/request.go
index 973dfa642a..c6fd28a5de 100644
--- a/pkg/protocols/headless/request.go
+++ b/pkg/protocols/headless/request.go
@@ -174,7 +174,7 @@ func (request *Request) executeRequestWithPayloads(input *contextargs.Context, p
reqBuilder.WriteString("\t" + actStepStr + "\n")
}
}
- gologger.Debug().Msgf(reqBuilder.String())
+ gologger.Debug().Msg(reqBuilder.String())
}
var responseBody string
diff --git a/pkg/reporting/reporting.go b/pkg/reporting/reporting.go
index 172c31105e..889f92f3f7 100644
--- a/pkg/reporting/reporting.go
+++ b/pkg/reporting/reporting.go
@@ -240,7 +240,7 @@ func (c *ReportingClient) Close() {
if failed > 0 {
msgBuilder.WriteString(fmt.Sprintf(", %d failed", failed))
}
- gologger.Info().Msgf(msgBuilder.String())
+ gologger.Info().Msgf("%v", msgBuilder.String())
}
}
}
diff --git a/pkg/tmplexec/exec.go b/pkg/tmplexec/exec.go
index c510131da2..3d09f5e7a0 100644
--- a/pkg/tmplexec/exec.go
+++ b/pkg/tmplexec/exec.go
@@ -68,8 +68,10 @@ func (e *TemplateExecuter) Compile() error {
if cliOptions.Verbose {
rawErrorMessage := dslCompilationError.Error()
formattedErrorMessage := strings.ToUpper(rawErrorMessage[:1]) + rawErrorMessage[1:] + "."
- gologger.Warning().Msgf(formattedErrorMessage)
+
+ gologger.Warning().Msg(formattedErrorMessage)
gologger.Info().Msgf("The available custom DSL functions are:")
+
fmt.Println(dsl.GetPrintableDslFunctionSignatures(cliOptions.NoColor))
}
}
From f2f250738c9baf258c6da8f81ae5d863d6da0043 Mon Sep 17 00:00:00 2001
From: sandeep <8293321+ehsandeep@users.noreply.github.com>
Date: Fri, 16 Aug 2024 20:44:39 +0530
Subject: [PATCH 148/283] version update
---
README.md | 1 +
pkg/catalog/config/constants.go | 2 +-
2 files changed, 2 insertions(+), 1 deletion(-)
diff --git a/README.md b/README.md
index b6217a9a72..412d348645 100644
--- a/README.md
+++ b/README.md
@@ -311,6 +311,7 @@ STATISTICS:
CLOUD:
-auth configure projectdiscovery cloud (pdcp) api key (default true)
+ -tid, -team-id string upload scan results to given team id (optional) (default "none")
-cup, -cloud-upload upload scan results to pdcp dashboard
-sid, -scan-id string upload scan results to existing scan id (optional)
-sname, -scan-name string scan name to set (optional)
diff --git a/pkg/catalog/config/constants.go b/pkg/catalog/config/constants.go
index 8aa6beedfe..cd0e5673c7 100644
--- a/pkg/catalog/config/constants.go
+++ b/pkg/catalog/config/constants.go
@@ -31,7 +31,7 @@ const (
CLIConfigFileName = "config.yaml"
ReportingConfigFilename = "reporting-config.yaml"
// Version is the current version of nuclei
- Version = `v3.3.0`
+ Version = `v3.3.1`
// Directory Names of custom templates
CustomS3TemplatesDirName = "s3"
CustomGitHubTemplatesDirName = "github"
From 02fadbcf775a0a2c8b0ee6eb3f4796c32e2c64ea Mon Sep 17 00:00:00 2001
From: Dwi Siswanto
Date: Mon, 19 Aug 2024 05:58:30 +0700
Subject: [PATCH 149/283] chore: migrate issue template to issue form (#5538)
Signed-off-by: Dwi Siswanto
---
.github/ISSUE_TEMPLATE/bug-report.yml | 72 ++++++++++++++++++++++
.github/ISSUE_TEMPLATE/feature-request.yml | 37 +++++++++++
.github/ISSUE_TEMPLATE/feature_request.md | 21 -------
.github/ISSUE_TEMPLATE/issue-report.md | 41 ------------
4 files changed, 109 insertions(+), 62 deletions(-)
create mode 100644 .github/ISSUE_TEMPLATE/bug-report.yml
create mode 100644 .github/ISSUE_TEMPLATE/feature-request.yml
delete mode 100644 .github/ISSUE_TEMPLATE/feature_request.md
delete mode 100644 .github/ISSUE_TEMPLATE/issue-report.md
diff --git a/.github/ISSUE_TEMPLATE/bug-report.yml b/.github/ISSUE_TEMPLATE/bug-report.yml
new file mode 100644
index 0000000000..d2053744c7
--- /dev/null
+++ b/.github/ISSUE_TEMPLATE/bug-report.yml
@@ -0,0 +1,72 @@
+name: Bug Report
+description: Create a report to help us to improve the Nuclei.
+title: "[BUG] ..."
+labels: ["Type: Bug"]
+body:
+ - type: markdown
+ attributes:
+ value: |
+ Thanks for taking the time to fill out this bug report!
+
+ For support requests, FAQs or "How to" questions, please use the [GitHub Discussions](https://github.com/projectdiscovery/nuclei/discussions) section instead or join our [Discord server](https://discord.gg/projectdiscovery) to discuss the idea on the **#nuclei** channel.
+
+ :warning: **Issues missing important information may be closed without further investigation.**
+ - type: checkboxes
+ attributes:
+ label: Is there an existing issue for this?
+ description: Please search to see if an issue already exists for the bug you encountered.
+ options:
+ - label: I have searched the existing issues.
+ required: true
+ - type: textarea
+ attributes:
+ label: Current Behavior
+ description: A concise description of what you're experiencing.
+ validations:
+ required: true
+ - type: textarea
+ attributes:
+ label: Expected Behavior
+ description: A concise description of what you expected to happen.
+ validations:
+ required: true
+ - type: textarea
+ attributes:
+ label: Steps To Reproduce
+ description: |
+ Steps to reproduce the behavior, for example, commands to run Nuclei.
+
+ :warning: **Please redact any literal target hosts/URLs or other sensitive information.**
+ placeholder: |
+ 1. Run `nuclei -t ...`
+ validations:
+ required: true
+ - type: textarea
+ attributes:
+ label: Relevant log output
+ description: Please copy and paste any relevant log output. This will be automatically formatted into code, so no need for backticks.
+ render: shell
+ - type: textarea
+ attributes:
+ label: Environment
+ description: |
+ Examples:
+ - **OS**: Ubuntu 20.04
+ - **Nuclei** (`nuclei -version`): v3.3.1
+ - **Go** (`go version`): go1.22.0 _(only if you've installed it via the `go install` command)_
+ value: |
+ - OS:
+ - Nuclei:
+ - Go:
+ render: markdown
+ validations:
+ required: true
+ - type: textarea
+ attributes:
+ label: Anything else?
+ description: |
+ Links? References? Templates? Anything that will give us more context about the issue you are encountering!
+
+ Tip: You can attach images or log files by clicking this area to highlight it and then dragging files in.
+ validations:
+ required: false
diff --git a/.github/ISSUE_TEMPLATE/feature-request.yml b/.github/ISSUE_TEMPLATE/feature-request.yml
new file mode 100644
index 0000000000..ccc1b86298
--- /dev/null
+++ b/.github/ISSUE_TEMPLATE/feature-request.yml
@@ -0,0 +1,37 @@
+name: Feature Request
+description: Request feature to implement in the Nuclei.
+title: "[FEATURE] ..."
+labels: ["Type: Enhancement"]
+body:
+ - type: markdown
+ attributes:
+ value: |
+ Thanks for taking the time to fill out this feature request!
+
+ Please make sure to provide a detailed description with all the relevant information that might be required to start working on this feature. In case you are not sure about your request or whether the particular feature is already supported or not, please [start a discussion](https://github.com/projectdiscovery/nuclei/discussions/categories/ideas) instead.
+
+ Join our [Discord server](https://discord.gg/projectdiscovery) to discuss the idea on the **#nuclei** channel.
+ - type: textarea
+ attributes:
+ label: Describe your feature request
+ description: A clear and concise description of feature to implement.
+ validations:
+ required: true
+ - type: textarea
+ attributes:
+ label: Describe the use case of the feature
+ description: A clear and concise description of the feature request's motivation and the use-cases in which it could be useful.
+ validations:
+ required: true
+ - type: textarea
+ attributes:
+ label: Describe alternatives you've considered
+ description: A clear and concise description of any alternative solutions or features you've considered.
+ validations:
+ required: false
+ - type: textarea
+ attributes:
+ label: Additional context
+ description: Add any other context about the feature request here.
+ validations:
+ required: false
diff --git a/.github/ISSUE_TEMPLATE/feature_request.md b/.github/ISSUE_TEMPLATE/feature_request.md
deleted file mode 100644
index 8b41d50c2a..0000000000
--- a/.github/ISSUE_TEMPLATE/feature_request.md
+++ /dev/null
@@ -1,21 +0,0 @@
----
-name: Feature request
-about: Request feature to implement in this project
-title: ""
-labels: 'Type: Enhancement'
-assignees: ''
-
----
-
-
-
-### Please describe your feature request:
-
-
-### Describe the use case of this feature:
-
diff --git a/.github/ISSUE_TEMPLATE/issue-report.md b/.github/ISSUE_TEMPLATE/issue-report.md
deleted file mode 100644
index 3aa563b195..0000000000
--- a/.github/ISSUE_TEMPLATE/issue-report.md
+++ /dev/null
@@ -1,41 +0,0 @@
----
-name: Issue report
-about: Create a report to help us to improve the project
-labels: 'Type: Bug'
-
----
-
-
-
-
-
-### Nuclei version:
-
-
-
-
-## Nuclei command:
-
-
-
-
-### Current Behavior:
-
-
-### Expected Behavior:
-
-
-### Steps To Reproduce:
-
-
-
-### Anything else:
-
From 80b3cc0627482adc17ce2822ae9368dd9d22267d Mon Sep 17 00:00:00 2001
From: Ice3man
Date: Mon, 19 Aug 2024 06:20:12 +0530
Subject: [PATCH 150/283] feat: jira accept issue-type and project ids (#5537)
* feat: jira accept issue-type and project ids
* remove validation for project name
---------
Co-authored-by: sandeep <8293321+ehsandeep@users.noreply.github.com>
---
pkg/reporting/trackers/jira/jira.go | 13 ++++++++++++-
1 file changed, 12 insertions(+), 1 deletion(-)
diff --git a/pkg/reporting/trackers/jira/jira.go b/pkg/reporting/trackers/jira/jira.go
index 5499445c87..6cb976c44e 100644
--- a/pkg/reporting/trackers/jira/jira.go
+++ b/pkg/reporting/trackers/jira/jira.go
@@ -70,9 +70,13 @@ type Options struct {
// Token is the token for jira instance.
Token string `yaml:"token" json:"token" validate:"required"`
// ProjectName is the name of the project.
- ProjectName string `yaml:"project-name" json:"project_name" validate:"required"`
+ ProjectName string `yaml:"project-name" json:"project_name"`
+ // ProjectID is the ID of the project (optional)
+ ProjectID string `yaml:"project-id" json:"project_id"`
// IssueType (optional) is the name of the created issue type
IssueType string `yaml:"issue-type" json:"issue_type"`
+ // IssueTypeID (optional) is the ID of the created issue type
+ IssueTypeID string `yaml:"issue-type-id" json:"issue_type_id"`
// SeverityAsLabel (optional) sends the severity as the label of the created
// issue.
SeverityAsLabel bool `yaml:"severity-as-label" json:"severity_as_label"`
@@ -183,6 +187,7 @@ func (i *Integration) CreateNewIssue(event *output.ResultEvent) (*filters.Create
Project: jira.Project{Key: i.options.ProjectName},
Summary: summary,
}
+
// On-prem version of Jira server does not use AccountID
if !i.options.Cloud {
fields = &jira.IssueFields{
@@ -195,6 +200,12 @@ func (i *Integration) CreateNewIssue(event *output.ResultEvent) (*filters.Create
Unknowns: customFields,
}
}
+ if i.options.IssueTypeID != "" {
+ fields.Type = jira.IssueType{ID: i.options.IssueTypeID}
+ }
+ if i.options.ProjectID != "" {
+ fields.Project = jira.Project{ID: i.options.ProjectID}
+ }
issueData := &jira.Issue{
Fields: fields,
From c6bafd88eb763b0e0787e3ff692e34f40df5fb36 Mon Sep 17 00:00:00 2001
From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com>
Date: Mon, 19 Aug 2024 05:12:33 +0000
Subject: [PATCH 151/283] chore(deps): bump
github.com/projectdiscovery/clistats
Bumps [github.com/projectdiscovery/clistats](https://github.com/projectdiscovery/clistats) from 0.0.20 to 0.1.0.
- [Release notes](https://github.com/projectdiscovery/clistats/releases)
- [Commits](https://github.com/projectdiscovery/clistats/compare/v0.0.20...v0.1.0)
---
updated-dependencies:
- dependency-name: github.com/projectdiscovery/clistats
dependency-type: direct:production
update-type: version-update:semver-minor
...
Signed-off-by: dependabot[bot]
---
go.mod | 2 +-
go.sum | 4 ++--
2 files changed, 3 insertions(+), 3 deletions(-)
diff --git a/go.mod b/go.mod
index f2053614c1..c52eb896a3 100644
--- a/go.mod
+++ b/go.mod
@@ -19,7 +19,7 @@ require (
github.com/miekg/dns v1.1.59
github.com/olekukonko/tablewriter v0.0.5
github.com/pkg/errors v0.9.1
- github.com/projectdiscovery/clistats v0.0.20
+ github.com/projectdiscovery/clistats v0.1.0
github.com/projectdiscovery/fastdialer v0.2.3
github.com/projectdiscovery/hmap v0.0.54
github.com/projectdiscovery/interactsh v1.2.0
diff --git a/go.sum b/go.sum
index dadb721a3f..c18b6f1f31 100644
--- a/go.sum
+++ b/go.sum
@@ -830,8 +830,8 @@ github.com/projectdiscovery/blackrock v0.0.1 h1:lHQqhaaEFjgf5WkuItbpeCZv2DUIE45k
github.com/projectdiscovery/blackrock v0.0.1/go.mod h1:ANUtjDfaVrqB453bzToU+YB4cUbvBRpLvEwoWIwlTss=
github.com/projectdiscovery/cdncheck v1.1.0 h1:qDITidmJsejzpk3rMkauCh6sjI2GH9hW/snk0cQ3kXE=
github.com/projectdiscovery/cdncheck v1.1.0/go.mod h1:sZ8U4MjHSsyaTVjBbYWHT1cwUVvUYwDX1W+WvWRicIc=
-github.com/projectdiscovery/clistats v0.0.20 h1:5jO5SLiRJ7f0nDV0ndBNmBeesbROouPooH+DGMgoWq4=
-github.com/projectdiscovery/clistats v0.0.20/go.mod h1:GJ2av0KnOvK0AISQnP8hyDclYIji1LVkx2l0pwnzAu4=
+github.com/projectdiscovery/clistats v0.1.0 h1:b+LF1w0xhNd7cneKWMXb+/yUmF1n5szawxP4XGpmbxs=
+github.com/projectdiscovery/clistats v0.1.0/go.mod h1:GJ2av0KnOvK0AISQnP8hyDclYIji1LVkx2l0pwnzAu4=
github.com/projectdiscovery/dsl v0.1.8 h1:ulz+o097XsYgqP4QXaAQhVSkoeD2ZPWd29wX3CTodUA=
github.com/projectdiscovery/dsl v0.1.8/go.mod h1:AYJS2WQ/q0smr2v4pEJTg4DPe6k56KFKR7UFXvzNz/4=
github.com/projectdiscovery/fastdialer v0.2.3 h1:K03x5XEXGyVWxS2rtSR104E9kHF0aphN7kOCzbh8zv0=
From e16350cde610e11509ebed808f1277ac83e75492 Mon Sep 17 00:00:00 2001
From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com>
Date: Mon, 19 Aug 2024 05:39:12 +0000
Subject: [PATCH 152/283] chore(deps): bump github.com/projectdiscovery/hmap
from 0.0.54 to 0.0.55
Bumps [github.com/projectdiscovery/hmap](https://github.com/projectdiscovery/hmap) from 0.0.54 to 0.0.55.
- [Release notes](https://github.com/projectdiscovery/hmap/releases)
- [Commits](https://github.com/projectdiscovery/hmap/compare/v0.0.54...v0.0.55)
---
updated-dependencies:
- dependency-name: github.com/projectdiscovery/hmap
dependency-type: direct:production
update-type: version-update:semver-patch
...
Signed-off-by: dependabot[bot]
---
go.mod | 8 ++++----
go.sum | 16 ++++++++--------
2 files changed, 12 insertions(+), 12 deletions(-)
diff --git a/go.mod b/go.mod
index c52eb896a3..c7628b7b47 100644
--- a/go.mod
+++ b/go.mod
@@ -21,11 +21,11 @@ require (
github.com/pkg/errors v0.9.1
github.com/projectdiscovery/clistats v0.1.0
github.com/projectdiscovery/fastdialer v0.2.3
- github.com/projectdiscovery/hmap v0.0.54
+ github.com/projectdiscovery/hmap v0.0.55
github.com/projectdiscovery/interactsh v1.2.0
github.com/projectdiscovery/rawhttp v0.1.60
- github.com/projectdiscovery/retryabledns v1.0.69
- github.com/projectdiscovery/retryablehttp-go v1.0.71
+ github.com/projectdiscovery/retryabledns v1.0.70
+ github.com/projectdiscovery/retryablehttp-go v1.0.72
github.com/projectdiscovery/yamldoc-go v1.0.4
github.com/remeh/sizedwaitgroup v1.0.0
github.com/rs/xid v1.5.0
@@ -96,7 +96,7 @@ require (
github.com/projectdiscovery/tlsx v1.1.6
github.com/projectdiscovery/uncover v1.0.9
github.com/projectdiscovery/useragent v0.0.65
- github.com/projectdiscovery/utils v0.2.4
+ github.com/projectdiscovery/utils v0.2.5
github.com/projectdiscovery/wappalyzergo v0.1.14
github.com/redis/go-redis/v9 v9.1.0
github.com/seh-msft/burpxml v1.0.1
diff --git a/go.sum b/go.sum
index c18b6f1f31..ecf4846ac7 100644
--- a/go.sum
+++ b/go.sum
@@ -850,8 +850,8 @@ github.com/projectdiscovery/gostruct v0.0.2 h1:s8gP8ApugGM4go1pA+sVlPDXaWqNP5BBD
github.com/projectdiscovery/gostruct v0.0.2/go.mod h1:H86peL4HKwMXcQQtEa6lmC8FuD9XFt6gkNR0B/Mu5PE=
github.com/projectdiscovery/gozero v0.0.2 h1:8fJeaCjxL9tpm33uG/RsCQs6HGM/NE6eA3cjkilRQ+E=
github.com/projectdiscovery/gozero v0.0.2/go.mod h1:d8bZvDWW07LWNYWrwjZ4OO1I0cpkfqaysyDfSs9ibK8=
-github.com/projectdiscovery/hmap v0.0.54 h1:b3pdQZwCw4is3xiL2jBx7SJZcYaf/7vtozY7bjUzO/s=
-github.com/projectdiscovery/hmap v0.0.54/go.mod h1:j0oakxYOWEfk29wRq5gQgrCv1JnfAfzGaMsRWwEas80=
+github.com/projectdiscovery/hmap v0.0.55 h1:By0qSeQRE/ybeITUEL6oURN+j7qX7FVjCq92duwsqcA=
+github.com/projectdiscovery/hmap v0.0.55/go.mod h1:uZOF/W8yOGDb/t9omqATirhVwiboa5F/c00OVhDeE0E=
github.com/projectdiscovery/httpx v1.6.7 h1:luJAtQ+iKpiyKKDpcWw1/fF1MaRX6JZ4R3P+ol5Uikk=
github.com/projectdiscovery/httpx v1.6.7/go.mod h1:rqcuexBrb4v36ZyX0BHcHaovTurOJn2P65Tdt9cYdfE=
github.com/projectdiscovery/interactsh v1.2.0 h1:Al6jHiR+Usl9egYJDLJaWNHOcH8Rugk8gWMasc8Cmw8=
@@ -872,10 +872,10 @@ github.com/projectdiscovery/rawhttp v0.1.60 h1:L57kAATGRuCfOp8Fs0ly3j2wlip7R5pco
github.com/projectdiscovery/rawhttp v0.1.60/go.mod h1:M97w6lsphTHKeqLSio4Lw9K4KLO2riJMYAOqVMIZ3ck=
github.com/projectdiscovery/rdap v0.9.1-0.20221108103045-9865884d1917 h1:m03X4gBVSorSzvmm0bFa7gDV4QNSOWPL/fgZ4kTXBxk=
github.com/projectdiscovery/rdap v0.9.1-0.20221108103045-9865884d1917/go.mod h1:JxXtZC9e195awe7EynrcnBJmFoad/BNDzW9mzFkK8Sg=
-github.com/projectdiscovery/retryabledns v1.0.69 h1:6vhhlfxPQoJI0j1enYSzyagyAZJNfGSg9TNloVsxEBQ=
-github.com/projectdiscovery/retryabledns v1.0.69/go.mod h1:5SDuGtXgL4W1z/9V+xnmHFYg03Fi1Ud/F2kqlPlAgfE=
-github.com/projectdiscovery/retryablehttp-go v1.0.71 h1:yXPNShCOwoTz7dBSJsBhBh4g4ujX62XS/BrH/fL1VyE=
-github.com/projectdiscovery/retryablehttp-go v1.0.71/go.mod h1:wY3T89EwcCKAw6iyMDvwzGPyL3d8TaBU80hnDErEKgM=
+github.com/projectdiscovery/retryabledns v1.0.70 h1:2yFMqQ4v3tgI9ORjlPH60h5QIs2EXxHVGCaxrOA1ZlI=
+github.com/projectdiscovery/retryabledns v1.0.70/go.mod h1:Ld/RLVsG7d+wlNcye9xcuPTjGHLF9XO8w34GLRKsNis=
+github.com/projectdiscovery/retryablehttp-go v1.0.72 h1:3m+9aRwC4KOMoQiOF3lVsDnPm/1+OR5r0UCHt5Edz5k=
+github.com/projectdiscovery/retryablehttp-go v1.0.72/go.mod h1:tDPEpm0PlDOMB4yqHFPoLJZaK3uo+Auj/QIiUlzS+6Y=
github.com/projectdiscovery/sarif v0.0.1 h1:C2Tyj0SGOKbCLgHrx83vaE6YkzXEVrMXYRGLkKCr/us=
github.com/projectdiscovery/sarif v0.0.1/go.mod h1:cEYlDu8amcPf6b9dSakcz2nNnJsoz4aR6peERwV+wuQ=
github.com/projectdiscovery/stringsutil v0.0.2 h1:uzmw3IVLJSMW1kEg8eCStG/cGbYYZAja8BH3LqqJXMA=
@@ -886,8 +886,8 @@ github.com/projectdiscovery/uncover v1.0.9 h1:s5RbkD/V4r8QcPkys4gTTqMuRSgXq0Jpre
github.com/projectdiscovery/uncover v1.0.9/go.mod h1:2PUF3SpB5QNIJ8epaB2xbRzkPaxEAWRDm3Ir2ijt81U=
github.com/projectdiscovery/useragent v0.0.65 h1:x78ZwWdqpzokOHxLITUXvq+ljkTKc19z3ILGtoV1N70=
github.com/projectdiscovery/useragent v0.0.65/go.mod h1:deOP8YLJU6SCzM8k+K8PjkcOF4Ux0spqyO4ODZGIT4A=
-github.com/projectdiscovery/utils v0.2.4 h1:CHnlt2la4jr8TeL7ZK7UhQItHY7DDXqIuLnnxyAJLDY=
-github.com/projectdiscovery/utils v0.2.4/go.mod h1:2Vx7geSrBfCPqknZywqbChQm8SE30mcyrlB5YsxEnhA=
+github.com/projectdiscovery/utils v0.2.5 h1:puiw3/gK1WTEj9ZKnW00tAS0ex+ETqbLJ8juRuejA98=
+github.com/projectdiscovery/utils v0.2.5/go.mod h1:fv3WC6m/+bVT+wH+fMB+JsKviEC/UAuDSeKxWPlK3kk=
github.com/projectdiscovery/wappalyzergo v0.1.14 h1:nt1IM4RUmqeymsXk4h6BsZbKDoS2hjFvPkT2GaI1rz4=
github.com/projectdiscovery/wappalyzergo v0.1.14/go.mod h1:/hzgxkBFTMe2wDbA93nFfoMjULw7/vIZ9QPSAnCgUa8=
github.com/projectdiscovery/yamldoc-go v1.0.4 h1:eZoESapnMw6WAHiVgRwNqvbJEfNHEH148uthhFbG5jE=
From c94d226a496a9b08026c5f39540356d772d48aba Mon Sep 17 00:00:00 2001
From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com>
Date: Mon, 19 Aug 2024 06:07:14 +0000
Subject: [PATCH 153/283] chore(deps): bump
github.com/projectdiscovery/fastdialer
Bumps [github.com/projectdiscovery/fastdialer](https://github.com/projectdiscovery/fastdialer) from 0.2.3 to 0.2.5.
- [Release notes](https://github.com/projectdiscovery/fastdialer/releases)
- [Commits](https://github.com/projectdiscovery/fastdialer/compare/v0.2.3...v0.2.5)
---
updated-dependencies:
- dependency-name: github.com/projectdiscovery/fastdialer
dependency-type: direct:production
update-type: version-update:semver-patch
...
Signed-off-by: dependabot[bot]
---
go.mod | 4 ++--
go.sum | 8 ++++----
2 files changed, 6 insertions(+), 6 deletions(-)
diff --git a/go.mod b/go.mod
index c7628b7b47..171d1bc523 100644
--- a/go.mod
+++ b/go.mod
@@ -20,11 +20,11 @@ require (
github.com/olekukonko/tablewriter v0.0.5
github.com/pkg/errors v0.9.1
github.com/projectdiscovery/clistats v0.1.0
- github.com/projectdiscovery/fastdialer v0.2.3
+ github.com/projectdiscovery/fastdialer v0.2.5
github.com/projectdiscovery/hmap v0.0.55
github.com/projectdiscovery/interactsh v1.2.0
github.com/projectdiscovery/rawhttp v0.1.60
- github.com/projectdiscovery/retryabledns v1.0.70
+ github.com/projectdiscovery/retryabledns v1.0.71
github.com/projectdiscovery/retryablehttp-go v1.0.72
github.com/projectdiscovery/yamldoc-go v1.0.4
github.com/remeh/sizedwaitgroup v1.0.0
diff --git a/go.sum b/go.sum
index ecf4846ac7..9bddf93345 100644
--- a/go.sum
+++ b/go.sum
@@ -834,8 +834,8 @@ github.com/projectdiscovery/clistats v0.1.0 h1:b+LF1w0xhNd7cneKWMXb+/yUmF1n5szaw
github.com/projectdiscovery/clistats v0.1.0/go.mod h1:GJ2av0KnOvK0AISQnP8hyDclYIji1LVkx2l0pwnzAu4=
github.com/projectdiscovery/dsl v0.1.8 h1:ulz+o097XsYgqP4QXaAQhVSkoeD2ZPWd29wX3CTodUA=
github.com/projectdiscovery/dsl v0.1.8/go.mod h1:AYJS2WQ/q0smr2v4pEJTg4DPe6k56KFKR7UFXvzNz/4=
-github.com/projectdiscovery/fastdialer v0.2.3 h1:K03x5XEXGyVWxS2rtSR104E9kHF0aphN7kOCzbh8zv0=
-github.com/projectdiscovery/fastdialer v0.2.3/go.mod h1:a0BKvETrO1EAohUYp9gwtbbce0hKD1qGrTegyAUqyRo=
+github.com/projectdiscovery/fastdialer v0.2.5 h1:OZg31LFIBBHbXVnOEcFNtD0gBM80zpY68QvYI0xbw80=
+github.com/projectdiscovery/fastdialer v0.2.5/go.mod h1:E1vmos7AeD/KZ7rsDeImjLp5H3jjas+2utVdUxGulvY=
github.com/projectdiscovery/fasttemplate v0.0.2 h1:h2cISk5xDhlJEinlBQS6RRx0vOlOirB2y3Yu4PJzpiA=
github.com/projectdiscovery/fasttemplate v0.0.2/go.mod h1:XYWWVMxnItd+r0GbjA1GCsUopMw1/XusuQxdyAIHMCw=
github.com/projectdiscovery/freeport v0.0.5 h1:jnd3Oqsl4S8n0KuFkE5Hm8WGDP24ITBvmyw5pFTHS8Q=
@@ -872,8 +872,8 @@ github.com/projectdiscovery/rawhttp v0.1.60 h1:L57kAATGRuCfOp8Fs0ly3j2wlip7R5pco
github.com/projectdiscovery/rawhttp v0.1.60/go.mod h1:M97w6lsphTHKeqLSio4Lw9K4KLO2riJMYAOqVMIZ3ck=
github.com/projectdiscovery/rdap v0.9.1-0.20221108103045-9865884d1917 h1:m03X4gBVSorSzvmm0bFa7gDV4QNSOWPL/fgZ4kTXBxk=
github.com/projectdiscovery/rdap v0.9.1-0.20221108103045-9865884d1917/go.mod h1:JxXtZC9e195awe7EynrcnBJmFoad/BNDzW9mzFkK8Sg=
-github.com/projectdiscovery/retryabledns v1.0.70 h1:2yFMqQ4v3tgI9ORjlPH60h5QIs2EXxHVGCaxrOA1ZlI=
-github.com/projectdiscovery/retryabledns v1.0.70/go.mod h1:Ld/RLVsG7d+wlNcye9xcuPTjGHLF9XO8w34GLRKsNis=
+github.com/projectdiscovery/retryabledns v1.0.71 h1:hZnTR7C4G7XDLxQEvdZVREHEJToIfeObNwR+zpmZ1vs=
+github.com/projectdiscovery/retryabledns v1.0.71/go.mod h1:P3GAytlIf2c/H+PbHGwLoPrc8pSsCK+i8gt3sO/VynU=
github.com/projectdiscovery/retryablehttp-go v1.0.72 h1:3m+9aRwC4KOMoQiOF3lVsDnPm/1+OR5r0UCHt5Edz5k=
github.com/projectdiscovery/retryablehttp-go v1.0.72/go.mod h1:tDPEpm0PlDOMB4yqHFPoLJZaK3uo+Auj/QIiUlzS+6Y=
github.com/projectdiscovery/sarif v0.0.1 h1:C2Tyj0SGOKbCLgHrx83vaE6YkzXEVrMXYRGLkKCr/us=
From 091ca32b1cd9f1190847e5506b086d3c35b1e0e1 Mon Sep 17 00:00:00 2001
From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com>
Date: Mon, 19 Aug 2024 06:33:12 +0000
Subject: [PATCH 154/283] chore(deps): bump
github.com/projectdiscovery/retryablehttp-go
Bumps [github.com/projectdiscovery/retryablehttp-go](https://github.com/projectdiscovery/retryablehttp-go) from 1.0.71 to 1.0.74.
- [Release notes](https://github.com/projectdiscovery/retryablehttp-go/releases)
- [Commits](https://github.com/projectdiscovery/retryablehttp-go/compare/v1.0.71...v1.0.74)
---
updated-dependencies:
- dependency-name: github.com/projectdiscovery/retryablehttp-go
dependency-type: direct:production
update-type: version-update:semver-patch
...
Signed-off-by: dependabot[bot]
---
go.mod | 2 +-
go.sum | 4 ++--
2 files changed, 3 insertions(+), 3 deletions(-)
diff --git a/go.mod b/go.mod
index 171d1bc523..c178fb5456 100644
--- a/go.mod
+++ b/go.mod
@@ -25,7 +25,7 @@ require (
github.com/projectdiscovery/interactsh v1.2.0
github.com/projectdiscovery/rawhttp v0.1.60
github.com/projectdiscovery/retryabledns v1.0.71
- github.com/projectdiscovery/retryablehttp-go v1.0.72
+ github.com/projectdiscovery/retryablehttp-go v1.0.74
github.com/projectdiscovery/yamldoc-go v1.0.4
github.com/remeh/sizedwaitgroup v1.0.0
github.com/rs/xid v1.5.0
diff --git a/go.sum b/go.sum
index 9bddf93345..4e74f6e6aa 100644
--- a/go.sum
+++ b/go.sum
@@ -874,8 +874,8 @@ github.com/projectdiscovery/rdap v0.9.1-0.20221108103045-9865884d1917 h1:m03X4gB
github.com/projectdiscovery/rdap v0.9.1-0.20221108103045-9865884d1917/go.mod h1:JxXtZC9e195awe7EynrcnBJmFoad/BNDzW9mzFkK8Sg=
github.com/projectdiscovery/retryabledns v1.0.71 h1:hZnTR7C4G7XDLxQEvdZVREHEJToIfeObNwR+zpmZ1vs=
github.com/projectdiscovery/retryabledns v1.0.71/go.mod h1:P3GAytlIf2c/H+PbHGwLoPrc8pSsCK+i8gt3sO/VynU=
-github.com/projectdiscovery/retryablehttp-go v1.0.72 h1:3m+9aRwC4KOMoQiOF3lVsDnPm/1+OR5r0UCHt5Edz5k=
-github.com/projectdiscovery/retryablehttp-go v1.0.72/go.mod h1:tDPEpm0PlDOMB4yqHFPoLJZaK3uo+Auj/QIiUlzS+6Y=
+github.com/projectdiscovery/retryablehttp-go v1.0.74 h1:jXZsg3nNDuDjGNeL0+hiEx/dqUqPyZW1zsotWW1y70c=
+github.com/projectdiscovery/retryablehttp-go v1.0.74/go.mod h1:fVRxa2fVMbomaPiOFA05kM2WrjV8wPFHW7J84eenVmQ=
github.com/projectdiscovery/sarif v0.0.1 h1:C2Tyj0SGOKbCLgHrx83vaE6YkzXEVrMXYRGLkKCr/us=
github.com/projectdiscovery/sarif v0.0.1/go.mod h1:cEYlDu8amcPf6b9dSakcz2nNnJsoz4aR6peERwV+wuQ=
github.com/projectdiscovery/stringsutil v0.0.2 h1:uzmw3IVLJSMW1kEg8eCStG/cGbYYZAja8BH3LqqJXMA=
From 8a3818230dcc074eba34dfab1234b5a117b4f31c Mon Sep 17 00:00:00 2001
From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com>
Date: Mon, 19 Aug 2024 06:59:19 +0000
Subject: [PATCH 155/283] chore(deps): bump
github.com/projectdiscovery/retryabledns
Bumps [github.com/projectdiscovery/retryabledns](https://github.com/projectdiscovery/retryabledns) from 1.0.69 to 1.0.72.
- [Release notes](https://github.com/projectdiscovery/retryabledns/releases)
- [Commits](https://github.com/projectdiscovery/retryabledns/compare/v1.0.69...v1.0.72)
---
updated-dependencies:
- dependency-name: github.com/projectdiscovery/retryabledns
dependency-type: direct:production
update-type: version-update:semver-patch
...
Signed-off-by: dependabot[bot]
---
go.mod | 2 +-
go.sum | 4 ++--
2 files changed, 3 insertions(+), 3 deletions(-)
diff --git a/go.mod b/go.mod
index c178fb5456..855b9f3c77 100644
--- a/go.mod
+++ b/go.mod
@@ -24,7 +24,7 @@ require (
github.com/projectdiscovery/hmap v0.0.55
github.com/projectdiscovery/interactsh v1.2.0
github.com/projectdiscovery/rawhttp v0.1.60
- github.com/projectdiscovery/retryabledns v1.0.71
+ github.com/projectdiscovery/retryabledns v1.0.72
github.com/projectdiscovery/retryablehttp-go v1.0.74
github.com/projectdiscovery/yamldoc-go v1.0.4
github.com/remeh/sizedwaitgroup v1.0.0
diff --git a/go.sum b/go.sum
index 4e74f6e6aa..43326744d8 100644
--- a/go.sum
+++ b/go.sum
@@ -872,8 +872,8 @@ github.com/projectdiscovery/rawhttp v0.1.60 h1:L57kAATGRuCfOp8Fs0ly3j2wlip7R5pco
github.com/projectdiscovery/rawhttp v0.1.60/go.mod h1:M97w6lsphTHKeqLSio4Lw9K4KLO2riJMYAOqVMIZ3ck=
github.com/projectdiscovery/rdap v0.9.1-0.20221108103045-9865884d1917 h1:m03X4gBVSorSzvmm0bFa7gDV4QNSOWPL/fgZ4kTXBxk=
github.com/projectdiscovery/rdap v0.9.1-0.20221108103045-9865884d1917/go.mod h1:JxXtZC9e195awe7EynrcnBJmFoad/BNDzW9mzFkK8Sg=
-github.com/projectdiscovery/retryabledns v1.0.71 h1:hZnTR7C4G7XDLxQEvdZVREHEJToIfeObNwR+zpmZ1vs=
-github.com/projectdiscovery/retryabledns v1.0.71/go.mod h1:P3GAytlIf2c/H+PbHGwLoPrc8pSsCK+i8gt3sO/VynU=
+github.com/projectdiscovery/retryabledns v1.0.72 h1:+k9JRYBMpZIyEQZJWVHoCHx1TEdpiuAS7iEru2bhs3A=
+github.com/projectdiscovery/retryabledns v1.0.72/go.mod h1:BDhYnqSH0Os2G37eb4L4vnKrUcik6Q6pR02Cq5PbT+0=
github.com/projectdiscovery/retryablehttp-go v1.0.74 h1:jXZsg3nNDuDjGNeL0+hiEx/dqUqPyZW1zsotWW1y70c=
github.com/projectdiscovery/retryablehttp-go v1.0.74/go.mod h1:fVRxa2fVMbomaPiOFA05kM2WrjV8wPFHW7J84eenVmQ=
github.com/projectdiscovery/sarif v0.0.1 h1:C2Tyj0SGOKbCLgHrx83vaE6YkzXEVrMXYRGLkKCr/us=
From 0da993afe6d41b4b1b814e8fad23a2acba13c60a Mon Sep 17 00:00:00 2001
From: Mzack9999
Date: Mon, 19 Aug 2024 14:32:54 +0200
Subject: [PATCH 156/283] Merge commit from fork
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit
* fix template signature verification
* fix signature pattern check
* add tests
* remove signature count constraint, check for lines len
* Add more tests
* Centralize signature extraction logic in signer package
* Move signature handling in Sign function to beginning
* Remove comment
* Revert `NewTemplateSigVerifier`
* update tests
* use ExtractSignatureAndContent func
* Allow signing code templates
* Remove unused const
---------
Co-authored-by: Doğan Can Bakır
Co-authored-by: Guy Goldenberg
---
pkg/templates/signer/tmpl_signer.go | 57 +++++-----
pkg/templates/signer/tmpl_signer_test.go | 126 +++++++++++++++++++++++
pkg/templates/template_sign.go | 3 +-
3 files changed, 161 insertions(+), 25 deletions(-)
create mode 100644 pkg/templates/signer/tmpl_signer_test.go
diff --git a/pkg/templates/signer/tmpl_signer.go b/pkg/templates/signer/tmpl_signer.go
index c4378fd406..a530da1fe3 100644
--- a/pkg/templates/signer/tmpl_signer.go
+++ b/pkg/templates/signer/tmpl_signer.go
@@ -11,7 +11,6 @@ import (
"errors"
"fmt"
"os"
- "regexp"
"strings"
"sync"
@@ -21,18 +20,21 @@ import (
)
var (
- ReDigest = regexp.MustCompile(`(?m)^#\sdigest:\s.+$`)
ErrUnknownAlgorithm = errors.New("unknown algorithm")
SignaturePattern = "# digest: "
SignatureFmt = SignaturePattern + "%x" + ":%v" // `#digest: :`
)
-func RemoveSignatureFromData(data []byte) []byte {
- return bytes.Trim(ReDigest.ReplaceAll(data, []byte("")), "\n")
-}
-
-func GetSignatureFromData(data []byte) []byte {
- return ReDigest.Find(data)
+// ExtractSignatureAndContent extracts the signature (if present) and returns the content without the signature
+func ExtractSignatureAndContent(data []byte) (signature, content []byte) {
+ dataStr := string(data)
+ if idx := strings.LastIndex(dataStr, SignaturePattern); idx != -1 {
+ signature = []byte(strings.TrimSpace(dataStr[idx:]))
+ content = []byte(strings.TrimSpace(dataStr[:idx]))
+ } else {
+ content = data
+ }
+ return
}
// SignableTemplate is a template that can be signed
@@ -69,26 +71,29 @@ func (t *TemplateSigner) GetUserFragment() string {
// Sign signs the given template with the template signer and returns the signature
func (t *TemplateSigner) Sign(data []byte, tmpl SignableTemplate) (string, error) {
+ existingSignature, content := ExtractSignatureAndContent(data)
+
// while re-signing template check if it has a code protocol
// if it does then verify that it is signed by current signer
// if not then return error
if tmpl.HasCodeProtocol() {
- sig := GetSignatureFromData(data)
- arr := strings.SplitN(string(sig), ":", 3)
- if len(arr) == 2 {
- // signature has no fragment
- return "", errorutil.NewWithTag("signer", "re-signing code templates are not allowed for security reasons.")
- }
- if len(arr) == 3 {
- // signature has fragment verify if it is equal to current fragment
- fragment := t.GetUserFragment()
- if fragment != arr[2] {
+ if len(existingSignature) > 0 {
+ arr := strings.SplitN(string(existingSignature), ":", 3)
+ if len(arr) == 2 {
+ // signature has no fragment
return "", errorutil.NewWithTag("signer", "re-signing code templates are not allowed for security reasons.")
}
+ if len(arr) == 3 {
+ // signature has fragment verify if it is equal to current fragment
+ fragment := t.GetUserFragment()
+ if fragment != arr[2] {
+ return "", errorutil.NewWithTag("signer", "re-signing code templates are not allowed for security reasons.")
+ }
+ }
}
}
- buff := bytes.NewBuffer(RemoveSignatureFromData(data))
+ buff := bytes.NewBuffer(content)
// if file has any imports process them
for _, file := range tmpl.GetFileImports() {
bin, err := os.ReadFile(file)
@@ -123,12 +128,16 @@ func (t *TemplateSigner) sign(data []byte) (string, error) {
// Verify verifies the given template with the template signer
func (t *TemplateSigner) Verify(data []byte, tmpl SignableTemplate) (bool, error) {
- digestData := ReDigest.Find(data)
- if len(digestData) == 0 {
- return false, errors.New("digest not found")
+ signature, content := ExtractSignatureAndContent(data)
+ if len(signature) == 0 {
+ return false, errors.New("no signature found")
+ }
+
+ if !bytes.HasPrefix(signature, []byte(SignaturePattern)) {
+ return false, errors.New("signature must be at the end of the template")
}
- digestData = bytes.TrimSpace(bytes.TrimPrefix(digestData, []byte(SignaturePattern)))
+ digestData := bytes.TrimSpace(bytes.TrimPrefix(signature, []byte(SignaturePattern)))
// remove fragment from digest as it is used for re-signing purposes only
digestString := strings.TrimSuffix(string(digestData), ":"+t.GetUserFragment())
digest, err := hex.DecodeString(digestString)
@@ -136,7 +145,7 @@ func (t *TemplateSigner) Verify(data []byte, tmpl SignableTemplate) (bool, error
return false, err
}
- buff := bytes.NewBuffer(RemoveSignatureFromData(data))
+ buff := bytes.NewBuffer(content)
// if file has any imports process them
for _, file := range tmpl.GetFileImports() {
bin, err := os.ReadFile(file)
diff --git a/pkg/templates/signer/tmpl_signer_test.go b/pkg/templates/signer/tmpl_signer_test.go
new file mode 100644
index 0000000000..832dcb0e05
--- /dev/null
+++ b/pkg/templates/signer/tmpl_signer_test.go
@@ -0,0 +1,126 @@
+package signer
+
+import (
+ "bytes"
+ "os"
+ "path/filepath"
+ "testing"
+
+ "github.com/stretchr/testify/assert"
+ "github.com/stretchr/testify/require"
+)
+
+const (
+ testCertFile = "../../../integration_tests/protocols/keys/ci.crt"
+ testKeyFile = "../../../integration_tests/protocols/keys/ci-private-key.pem"
+)
+
+type mockSignableTemplate struct {
+ imports []string
+ hasCode bool
+}
+
+func (m *mockSignableTemplate) GetFileImports() []string {
+ return m.imports
+}
+
+func (m *mockSignableTemplate) HasCodeProtocol() bool {
+ return m.hasCode
+}
+
+var signer, _ = NewTemplateSignerFromFiles(testCertFile, testKeyFile)
+
+func TestTemplateSignerSignAndVerify(t *testing.T) {
+ tempDir := t.TempDir()
+
+ tests := []struct {
+ name string
+ data []byte
+ tmpl SignableTemplate
+ wantSignErr bool
+ wantVerifyErr bool
+ wantVerified bool
+ modifyAfterSign func([]byte) []byte
+ }{
+ {
+ name: "Simple template",
+ data: []byte("id: test-template\ninfo:\n name: Test Template"),
+ tmpl: &mockSignableTemplate{},
+ wantVerified: true,
+ },
+ {
+ name: "Template with imports",
+ data: []byte("id: test-template\ninfo:\n name: Test Template"),
+ tmpl: &mockSignableTemplate{imports: []string{
+ filepath.Join(tempDir, "import1.yaml"),
+ filepath.Join(tempDir, "import2.yaml"),
+ }},
+ wantVerified: true,
+ },
+ {
+ name: "Template with code protocol",
+ data: []byte("id: test-template\ninfo:\n name: Test Template\n\ncode:\n - engine: bash\n source: echo 'Hello, World!'"),
+ tmpl: &mockSignableTemplate{hasCode: true},
+ wantSignErr: false,
+ wantVerified: true,
+ },
+ {
+ name: "Tampered template",
+ data: []byte("id: test-template\ninfo:\n name: Test Template"),
+ tmpl: &mockSignableTemplate{},
+ modifyAfterSign: func(data []byte) []byte {
+ signatureIndex := bytes.LastIndex(data, []byte(SignaturePattern))
+ if signatureIndex == -1 {
+ return data
+ }
+ return append(data[:signatureIndex], append([]byte("# Tampered content\n"), data[signatureIndex:]...)...)
+ },
+ wantVerified: false,
+ },
+ {
+ name: "Invalid signature",
+ data: []byte("id: test-template\ninfo:\n name: Test Template"),
+ tmpl: &mockSignableTemplate{},
+ modifyAfterSign: func(data []byte) []byte {
+ return append(bytes.TrimSuffix(data, []byte("\n")), []byte("\n# digest: invalid_signature:fragment")...)
+ },
+ wantVerifyErr: true,
+ wantVerified: false,
+ },
+ }
+
+ for _, tt := range tests {
+ t.Run(tt.name, func(t *testing.T) {
+ // Create import files if needed
+ for _, imp := range tt.tmpl.GetFileImports() {
+ err := os.WriteFile(imp, []byte("imported content"), 0644)
+ require.NoError(t, err, "Failed to create import file")
+ }
+
+ // Sign the template
+ signature, err := signer.Sign(tt.data, tt.tmpl)
+ if tt.wantSignErr {
+ assert.Error(t, err, "Expected an error during signing")
+ return
+ }
+ require.NoError(t, err, "Failed to sign template")
+
+ // Append signature to the template data
+ signedData := append(tt.data, []byte("\n"+signature)...)
+
+ // Apply any modifications after signing if specified
+ if tt.modifyAfterSign != nil {
+ signedData = tt.modifyAfterSign(signedData)
+ }
+
+ // Verify the signature
+ verified, err := signer.Verify(signedData, tt.tmpl)
+ if tt.wantVerifyErr {
+ assert.Error(t, err, "Expected an error during verification")
+ } else {
+ assert.NoError(t, err, "Unexpected error during verification")
+ }
+ assert.Equal(t, tt.wantVerified, verified, "Unexpected verification result")
+ })
+ }
+}
diff --git a/pkg/templates/template_sign.go b/pkg/templates/template_sign.go
index 24ae41c5e0..1eb09a447c 100644
--- a/pkg/templates/template_sign.go
+++ b/pkg/templates/template_sign.go
@@ -75,11 +75,12 @@ func SignTemplate(templateSigner *signer.TemplateSigner, templatePath string) er
return ErrNotATemplate
}
if !template.Verified {
+ _, content := signer.ExtractSignatureAndContent(bin)
signatureData, err := templateSigner.Sign(bin, template)
if err != nil {
return err
}
- buff := bytes.NewBuffer(signer.RemoveSignatureFromData(bin))
+ buff := bytes.NewBuffer(content)
buff.WriteString("\n" + signatureData)
return os.WriteFile(templatePath, buff.Bytes(), 0644)
}
From 3064788d35a8ec8623eb3a54eca920ea289f1259 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Do=C4=9Fan=20Can=20Bak=C4=B1r?=
Date: Mon, 19 Aug 2024 23:02:27 +0300
Subject: [PATCH 157/283] fix race condition
---
.../http/httpclientpool/clientpool.go | 20 +++++++++++++++++++
pkg/protocols/http/request.go | 5 +++--
2 files changed, 23 insertions(+), 2 deletions(-)
diff --git a/pkg/protocols/http/httpclientpool/clientpool.go b/pkg/protocols/http/httpclientpool/clientpool.go
index 2c244556ae..da7d5467e2 100644
--- a/pkg/protocols/http/httpclientpool/clientpool.go
+++ b/pkg/protocols/http/httpclientpool/clientpool.go
@@ -32,6 +32,7 @@ var (
forceMaxRedirects int
normalClient *retryablehttp.Client
clientPool *mapsutil.SyncLockMap[string, *retryablehttp.Client]
+ rawHttpClientMu sync.Mutex
)
// Init initializes the clientpool implementation
@@ -102,6 +103,22 @@ type Configuration struct {
ResponseHeaderTimeout time.Duration
}
+func (c *Configuration) Clone() *Configuration {
+ clone := *c
+ if c.Connection != nil {
+ cloneConnection := &ConnectionConfiguration{
+ DisableKeepAlive: c.Connection.DisableKeepAlive,
+ }
+ if c.Connection.HasCookieJar() {
+ cookiejar := *c.Connection.GetCookieJar()
+ cloneConnection.SetCookieJar(&cookiejar)
+ }
+ clone.Connection = cloneConnection
+ }
+
+ return &clone
+}
+
// Hash returns the hash of the configuration to allow client pooling
func (c *Configuration) Hash() string {
builder := &strings.Builder{}
@@ -131,6 +148,9 @@ func (c *Configuration) HasStandardOptions() bool {
// GetRawHTTP returns the rawhttp request client
func GetRawHTTP(options *protocols.ExecutorOptions) *rawhttp.Client {
+ rawHttpClientMu.Lock()
+ defer rawHttpClientMu.Unlock()
+
if rawHttpClient == nil {
rawHttpOptions := rawhttp.DefaultOptions
if types.ProxyURL != "" {
diff --git a/pkg/protocols/http/request.go b/pkg/protocols/http/request.go
index 98e1932477..3955325599 100644
--- a/pkg/protocols/http/request.go
+++ b/pkg/protocols/http/request.go
@@ -770,7 +770,7 @@ func (request *Request) executeRequest(input *contextargs.Context, generatedRequ
// check for cookie related configuration
if input.CookieJar != nil {
- connConfiguration := request.connConfiguration
+ connConfiguration := request.connConfiguration.Clone()
connConfiguration.Connection.SetCookieJar(input.CookieJar)
modifiedConfig = connConfiguration
}
@@ -778,7 +778,8 @@ func (request *Request) executeRequest(input *contextargs.Context, generatedRequ
updatedTimeout, ok := generatedRequest.request.Context().Value(httpclientpool.WithCustomTimeout{}).(httpclientpool.WithCustomTimeout)
if ok {
if modifiedConfig == nil {
- modifiedConfig = request.connConfiguration
+ connConfiguration := request.connConfiguration.Clone()
+ modifiedConfig = connConfiguration
}
modifiedConfig.ResponseHeaderTimeout = updatedTimeout.Timeout
}
From 50023428d0ae90f5ed09ebe4392364e06129ecb1 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Do=C4=9Fan=20Can=20Bak=C4=B1r?=
Date: Mon, 19 Aug 2024 23:09:07 +0300
Subject: [PATCH 158/283] fix lint
---
lib/sdk.go | 3 ++-
1 file changed, 2 insertions(+), 1 deletion(-)
diff --git a/lib/sdk.go b/lib/sdk.go
index 04b084f244..daeb68c14b 100644
--- a/lib/sdk.go
+++ b/lib/sdk.go
@@ -178,7 +178,8 @@ func (e *NucleiEngine) SignTemplate(tmplSigner *signer.TemplateSigner, data []by
if err != nil {
return data, err
}
- buff := bytes.NewBuffer(signer.RemoveSignatureFromData(data))
+ _, content := signer.ExtractSignatureAndContent(data)
+ buff := bytes.NewBuffer(content)
buff.WriteString("\n" + signatureData)
return buff.Bytes(), err
}
From d84aea93cc2844132af1ae364567124a48b93a87 Mon Sep 17 00:00:00 2001
From: Dwi Siswanto
Date: Tue, 20 Aug 2024 15:04:01 +0700
Subject: [PATCH 159/283] chore: sorts gitignore (#5548)
Signed-off-by: Dwi Siswanto
---
.gitignore | 73 +++++++++++++++++++++++++++---------------------------
1 file changed, 36 insertions(+), 37 deletions(-)
diff --git a/.gitignore b/.gitignore
index b9e561e3a6..5a50568910 100644
--- a/.gitignore
+++ b/.gitignore
@@ -1,43 +1,42 @@
+# Misc
+**/*-cache
+**/*-config
+**/.cache
+*.DS_Store
+*.exe
+.devcontainer
+.gitignore
.idea
.vscode
-.devcontainer
-vendor
+
+# Binaries
+**/bindgen
+**/debug-*
+**/docgen
+**/functional-test
+**/fuzzplayground
+**/integration-test
+**/jsdocgen
+**/main
+**/memogen
+**/nuclei
+**/nuclei-stats*
+**/nuclei_dev
+**/nuclei_main
+**/scan-charts
+**/scrapefunc
+**/scrapefuncs
+**/tsgen
+
+# Templates
+/*.yaml
+
+# Generated docs
+/dsl.md
+
+# Junk
dist
-integration_tests/nuclei
-integration_tests/.cache
-integration_tests/integration-test
-cmd/nuclei/main
-cmd/nuclei/nuclei
-cmd/integration-test/nuclei
-cmd/functional-test/nuclei_dev
-cmd/functional-test/nuclei_main
-cmd/functional-test/functional-test
-cmd/docgen/docgen
pkg/protocols/common/helpers/deserialization/testdata/Deserialize.class
pkg/protocols/common/helpers/deserialization/testdata/ValueObject.class
pkg/protocols/common/helpers/deserialization/testdata/ValueObject2.ser
-*.exe
-.gitignore
-pkg/js/devtools/bindgen/cmd/bindgen/bindgen
-pkg/js/devtools/jsdocgen/jsdocgen
-pkg/js/devtools/tsgen/tsgen
-pkg/js/devtools/tsgen/cmd/tsgen/tsgen
-*.DS_Store
-pkg/protocols/headless/engine/.cache
-/nuclei
-/bindgen
-/jsdocgen
-/tsgen
-/scrapefuncs
-/integration_tests/.cache/
-/*.yaml
-**/*-config
-**/*-cache
-/fuzzplayground
-integration_tests/fuzzplayground
-/dsl.md
-/nuclei-stats
-/nuclei-stats-*
-/scan-charts
-/**/debug-*
-
+vendor
\ No newline at end of file
From b1152efd422363fd00843cc1ff94b19837e54392 Mon Sep 17 00:00:00 2001
From: Dwi Siswanto
Date: Tue, 20 Aug 2024 15:06:39 +0700
Subject: [PATCH 160/283] chore(make): make it DRY (#5549)
Signed-off-by: Dwi Siswanto
---
Makefile | 141 ++++++++++++++++++++++++++++++++++++++++---------------
1 file changed, 102 insertions(+), 39 deletions(-)
diff --git a/Makefile b/Makefile
index a3f60a64b3..3fa6a033ef 100644
--- a/Makefile
+++ b/Makefile
@@ -1,64 +1,127 @@
# Go parameters
-GOCMD=go
-GOBUILD=$(GOCMD) build
-GOMOD=$(GOCMD) mod
-GOTEST=$(GOCMD) test
+GOCMD := go
+GOBUILD := $(GOCMD) build
+GOBUILD_OUTPUT :=
+GOBUILD_PACKAGES :=
+GOBUILD_ADDITIONAL_ARGS :=
+GOMOD := $(GOCMD) mod
+GOTEST := $(GOCMD) test
GOFLAGS := -v
# This should be disabled if the binary uses pprof
LDFLAGS := -s -w
ifneq ($(shell go env GOOS),darwin)
-LDFLAGS := -extldflags "-static"
+ LDFLAGS = -extldflags "-static"
endif
-.PHONY: all build build-stats scan-charts docs test integration functional tidy verify devtools jsupdate ts fuzzplayground memogen dsl-docs
+.PHONY: all build build-stats clean devtools-all devtools-bindgen devtools-scrapefuncs
+.PHONY: devtools-tsgen docs dsl-docs functional fuzzplayground go-build integration
+.PHONY: jsupdate-all jsupdate-bindgen jsupdate-tsgen memogen scan-charts test tidy ts verify
all: build
-build:
- rm -f nuclei 2>/dev/null
- $(GOBUILD) $(GOFLAGS) -ldflags '$(LDFLAGS)' -o "nuclei" cmd/nuclei/main.go
-build-stats:
- rm -f nuclei-stats 2>/dev/null
- $(GOBUILD) $(GOFLAGS) -ldflags '$(LDFLAGS)' -tags=stats -o "nuclei-stats" cmd/nuclei/main.go
-scan-charts:
- rm -f scan-charts 2>/dev/null
- $(GOBUILD) $(GOFLAGS) -ldflags '$(LDFLAGS)' -o "scan-charts" cmd/scan-charts/main.go
+
+clean:
+ rm -f '${GOBUILD_OUTPUT}' 2>/dev/null
+
+go-build: clean
+go-build:
+ $(GOBUILD) $(GOFLAGS) -ldflags '${LDFLAGS}' $(GOBUILD_ADDITIONAL_ARGS) \
+ -o '${GOBUILD_OUTPUT}' $(GOBUILD_PACKAGES)
+
+build: GOBUILD_OUTPUT = nuclei
+build: GOBUILD_PACKAGES = cmd/nuclei/main.go
+build: go-build
+
+build-stats: GOBUILD_OUTPUT = nuclei-stats
+build-stats: GOBUILD_PACKAGES = cmd/nuclei/main.go
+build-stats: GOBUILD_ADDITIONAL_ARGS = -tags=stats
+build-stats: go-build
+
+scan-charts: GOBUILD_OUTPUT = scan-charts
+scan-charts: GOBUILD_PACKAGES = cmd/scan-charts/main.go
+scan-charts: go-build
+
+docs: GOBUILD_OUTPUT = cmd/docgen/docgen
+docs: GOBUILD_PACKAGES = cmd/docgen/docgen.go
+docs: bin = dstdocgen
docs:
- if ! which dstdocgen > /dev/null; then
- echo -e "Command not found! Install? (y/n) \c"
- go get -v github.com/projectdiscovery/yamldoc-go/cmd/docgen/dstdocgen
+ @if ! which $(bin) >/dev/null; then \
+ read -p "${bin} not found. Do you want to install it? (y/n) " answer; \
+ if [ "$$answer" = "y" ]; then \
+ echo "Installing ${bin}..."; \
+ go get -v github.com/projectdiscovery/yamldoc-go/cmd/docgen/$(bin); \
+ go install -v github.com/projectdiscovery/yamldoc-go/cmd/docgen/$(bin); \
+ else \
+ echo "Please install ${bin} manually."; \
+ exit 1; \
+ fi \
fi
+
+ # TODO: Handle the panic, so that we just need to run `go install $(bin)@latest` (line 51-52)
$(GOCMD) generate pkg/templates/templates.go
- $(GOBUILD) -o "cmd/docgen/docgen" cmd/docgen/docgen.go
- ./cmd/docgen/docgen docs.md nuclei-jsonschema.json
+
+ $(GOBUILD) -o "${GOBUILD_OUTPUT}" $(GOBUILD_PACKAGES)
+ ./$(GOBUILD_OUTPUT) docs.md nuclei-jsonschema.json
+
+ git reset --hard # line 59
+
test:
$(GOTEST) $(GOFLAGS) ./...
+
integration:
cd integration_tests; bash run.sh
+
functional:
cd cmd/functional-test; bash run.sh
+
tidy:
$(GOMOD) tidy
+
verify: tidy
$(GOMOD) verify
-devtools:
- $(GOBUILD) $(GOFLAGS) -ldflags '$(LDFLAGS)' -o "bindgen" pkg/js/devtools/bindgen/cmd/bindgen/main.go
- $(GOBUILD) $(GOFLAGS) -ldflags '$(LDFLAGS)' -o "tsgen" pkg/js/devtools/tsgen/cmd/tsgen/main.go
- $(GOBUILD) $(GOFLAGS) -ldflags '$(LDFLAGS)' -o "scrapefuncs" pkg/js/devtools/scrapefuncs/main.go
-jsupdate:
- $(GOBUILD) $(GOFLAGS) -ldflags '$(LDFLAGS)' -o "bindgen" pkg/js/devtools/bindgen/cmd/bindgen/main.go
- $(GOBUILD) $(GOFLAGS) -ldflags '$(LDFLAGS)' -o "tsgen" pkg/js/devtools/tsgen/cmd/tsgen/main.go
- ./bindgen -dir pkg/js/libs -out pkg/js/generated
- ./tsgen -dir pkg/js/libs -out pkg/js/generated/ts
-ts:
- $(GOBUILD) $(GOFLAGS) -ldflags '$(LDFLAGS)' -o "tsgen" pkg/js/devtools/tsgen/cmd/tsgen/main.go
- ./tsgen -dir pkg/js/libs -out pkg/js/generated/ts
-fuzzplayground:
- $(GOBUILD) $(GOFLAGS) -ldflags '$(LDFLAGS)' -o "fuzzplayground" cmd/tools/fuzzplayground/main.go
+
+devtools-bindgen: GOBUILD_OUTPUT = bindgen
+devtools-bindgen: GOBUILD_PACKAGES = pkg/js/devtools/bindgen/cmd/bindgen/main.go
+devtools-bindgen: go-build
+
+devtools-tsgen: GOBUILD_OUTPUT = tsgen
+devtools-tsgen: GOBUILD_PACKAGES = pkg/js/devtools/tsgen/cmd/tsgen/main.go
+devtools-tsgen: go-build
+
+devtools-scrapefuncs: GOBUILD_OUTPUT = scrapefuncs
+devtools-scrapefuncs: GOBUILD_PACKAGES = pkg/js/devtools/scrapefuncs/main.go
+devtools-scrapefuncs: go-build
+
+devtools-all: devtools-bindgen devtools-tsgen devtools-scrapefuncs
+
+jsupdate-bindgen: GOBUILD_OUTPUT = bindgen
+jsupdate-bindgen: GOBUILD_PACKAGES = pkg/js/devtools/bindgen/cmd/bindgen/main.go
+jsupdate-bindgen: go-build
+jsupdate-bindgen:
+ ./$(GOBUILD_OUTPUT) -dir pkg/js/libs -out pkg/js/generated
+
+jsupdate-tsgen: GOBUILD_OUTPUT = tsgen
+jsupdate-tsgen: GOBUILD_PACKAGES = pkg/js/devtools/tsgen/cmd/tsgen/main.go
+jsupdate-tsgen: go-build
+jsupdate-tsgen:
+ ./$(GOBUILD_OUTPUT) -dir pkg/js/libs -out pkg/js/generated/ts
+
+jsupdate-all: jsupdate-bindgen jsupdate-tsgen
+
+ts: jsupdate-tsgen
+
+fuzzplayground: GOBUILD_OUTPUT = fuzzplayground
+fuzzplayground: GOBUILD_PACKAGES = cmd/tools/fuzzplayground/main.go
+fuzzplayground: LDFLAGS = -s -w
+fuzzplayground: go-build
+
+memogen: GOBUILD_OUTPUT = memogen
+memogen: GOBUILD_PACKAGES = cmd/memogen/memogen.go
+memogen: go-build
memogen:
- $(GOBUILD) $(GOFLAGS) -ldflags '$(LDFLAGS)' -o "memogen" cmd/memogen/memogen.go
- ./memogen -src pkg/js/libs -tpl cmd/memogen/function.tpl
+ ./$(GOBUILD_OUTPUT) -src pkg/js/libs -tpl cmd/memogen/function.tpl
+
+dsl-docs: GOBUILD_OUTPUT = scrapefuncs
+dsl-docs: GOBUILD_PACKAGES = pkg/js/devtools/scrapefuncs/main.go
dsl-docs:
- rm -f dsl.md scrapefuncs 2>/dev/null
- $(GOBUILD) $(GOFLAGS) -ldflags '$(LDFLAGS)' -o "scrapefuncs" pkg/js/devtools/scrapefuncs/main.go
- ./scrapefuncs -out dsl.md
+ ./$(GOBUILD_OUTPUT) -out dsl.md
From 46782ff90c29d2bc04ca130763ffbf95e95c5b37 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Do=C4=9Fan=20Can=20Bak=C4=B1r?=
Date: Wed, 21 Aug 2024 11:26:17 +0300
Subject: [PATCH 161/283] use sync.Once
---
pkg/protocols/http/httpclientpool/clientpool.go | 9 +++------
1 file changed, 3 insertions(+), 6 deletions(-)
diff --git a/pkg/protocols/http/httpclientpool/clientpool.go b/pkg/protocols/http/httpclientpool/clientpool.go
index da7d5467e2..5750d5aead 100644
--- a/pkg/protocols/http/httpclientpool/clientpool.go
+++ b/pkg/protocols/http/httpclientpool/clientpool.go
@@ -29,10 +29,10 @@ import (
var (
rawHttpClient *rawhttp.Client
+ rawHttpClientOnce sync.Once
forceMaxRedirects int
normalClient *retryablehttp.Client
clientPool *mapsutil.SyncLockMap[string, *retryablehttp.Client]
- rawHttpClientMu sync.Mutex
)
// Init initializes the clientpool implementation
@@ -148,10 +148,7 @@ func (c *Configuration) HasStandardOptions() bool {
// GetRawHTTP returns the rawhttp request client
func GetRawHTTP(options *protocols.ExecutorOptions) *rawhttp.Client {
- rawHttpClientMu.Lock()
- defer rawHttpClientMu.Unlock()
-
- if rawHttpClient == nil {
+ rawHttpClientOnce.Do(func() {
rawHttpOptions := rawhttp.DefaultOptions
if types.ProxyURL != "" {
rawHttpOptions.Proxy = types.ProxyURL
@@ -162,7 +159,7 @@ func GetRawHTTP(options *protocols.ExecutorOptions) *rawhttp.Client {
}
rawHttpOptions.Timeout = options.Options.GetTimeouts().HttpTimeout
rawHttpClient = rawhttp.NewClient(rawHttpOptions)
- }
+ })
return rawHttpClient
}
From 35a0d673ad8e12b11e90e8e0090feb26ea042b46 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Do=C4=9Fan=20Can=20Bak=C4=B1r?=
Date: Wed, 21 Aug 2024 11:36:33 +0300
Subject: [PATCH 162/283] remove redundant code
---
pkg/protocols/common/protocolstate/state.go | 1 -
1 file changed, 1 deletion(-)
diff --git a/pkg/protocols/common/protocolstate/state.go b/pkg/protocols/common/protocolstate/state.go
index b7c7796fa8..b21a1be9e5 100644
--- a/pkg/protocols/common/protocolstate/state.go
+++ b/pkg/protocols/common/protocolstate/state.go
@@ -214,6 +214,5 @@ func Close() {
Dialer.Close()
Dialer = nil
}
- Dialer = nil
StopActiveMemGuardian()
}
From 87c8cd8e0133d03f31ecad27f18f738c04174cb0 Mon Sep 17 00:00:00 2001
From: Roy Reznik
Date: Wed, 21 Aug 2024 11:44:01 +0100
Subject: [PATCH 163/283] Upgrade gitlab
---
go.mod | 10 ++++------
go.sum | 19 ++++++++-----------
pkg/reporting/trackers/gitlab/gitlab.go | 2 +-
3 files changed, 13 insertions(+), 18 deletions(-)
diff --git a/go.mod b/go.mod
index 855b9f3c77..026a96169e 100644
--- a/go.mod
+++ b/go.mod
@@ -36,10 +36,10 @@ require (
github.com/syndtr/goleveldb v1.0.0
github.com/valyala/fasttemplate v1.2.2
github.com/weppos/publicsuffix-go v0.30.2
- github.com/xanzy/go-gitlab v0.84.0
+ github.com/xanzy/go-gitlab v0.107.0
go.uber.org/multierr v1.11.0
golang.org/x/net v0.26.0
- golang.org/x/oauth2 v0.18.0
+ golang.org/x/oauth2 v0.22.0
golang.org/x/text v0.16.0
gopkg.in/yaml.v2 v2.4.0
)
@@ -262,7 +262,6 @@ require (
github.com/gobwas/pool v0.2.1 // indirect
github.com/golang-jwt/jwt/v4 v4.5.0 // indirect
github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da // indirect
- github.com/golang/protobuf v1.5.4 // indirect
github.com/golang/snappy v0.0.4 // indirect
github.com/google/go-querystring v1.1.0 // indirect
github.com/google/uuid v1.6.0 // indirect
@@ -306,10 +305,9 @@ require (
golang.org/x/exp v0.0.0-20240506185415-9bf2ced13842
golang.org/x/mod v0.17.0 // indirect
golang.org/x/sys v0.21.0 // indirect
- golang.org/x/time v0.5.0 // indirect
+ golang.org/x/time v0.6.0 // indirect
golang.org/x/tools v0.21.1-0.20240508182429-e35e4ccd0d2d
- google.golang.org/appengine v1.6.8 // indirect
- google.golang.org/protobuf v1.33.0 // indirect
+ google.golang.org/protobuf v1.34.2 // indirect
gopkg.in/alecthomas/kingpin.v2 v2.2.6 // indirect
gopkg.in/corvus-ch/zbase32.v1 v1.0.0 // indirect
)
diff --git a/go.sum b/go.sum
index 43326744d8..22a4618330 100644
--- a/go.sum
+++ b/go.sum
@@ -453,8 +453,6 @@ github.com/golang/protobuf v1.4.3/go.mod h1:oDoupMAO8OvCJWAcko0GGGIgR6R6ocIYbsSw
github.com/golang/protobuf v1.5.0/go.mod h1:FsONVRAS9T7sI+LIUmWTfcYkHO4aIWwzhcaSAoJOfIk=
github.com/golang/protobuf v1.5.2/go.mod h1:XVQd3VNwM+JqD3oG2Ue2ip4fOMUkwXdXDdiuN0vRsmY=
github.com/golang/protobuf v1.5.3/go.mod h1:XVQd3VNwM+JqD3oG2Ue2ip4fOMUkwXdXDdiuN0vRsmY=
-github.com/golang/protobuf v1.5.4 h1:i7eJL8qZTpSEXOPTxNKhASYpMn+8e5Q6AdndVa1dWek=
-github.com/golang/protobuf v1.5.4/go.mod h1:lnTiLA8Wa4RWRcIUkrtSVa5nRhsEGBg48fD6rSs7xps=
github.com/golang/snappy v0.0.0-20180518054509-2e65f85255db/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q=
github.com/golang/snappy v0.0.2/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q=
github.com/golang/snappy v0.0.4 h1:yAGX7huGHXlcLOEtBnF4w7FQwA26wojNCwOYAEhLjQM=
@@ -1068,8 +1066,8 @@ github.com/weppos/publicsuffix-go v0.30.2/go.mod h1:/hGscit36Yt+wammfBBwdMdxBT8b
github.com/weppos/publicsuffix-go/publicsuffix/generator v0.0.0-20220927085643-dc0d00c92642/go.mod h1:GHfoeIdZLdZmLjMlzBftbTDntahTttUMWjxZwQJhULE=
github.com/wk8/go-ordered-map/v2 v2.1.8 h1:5h/BUHu93oj4gIdvHHHGsScSTMijfx5PeYkE/fJgbpc=
github.com/wk8/go-ordered-map/v2 v2.1.8/go.mod h1:5nJHM5DyteebpVlHnWMV0rPz6Zp7+xBAnxjb1X5vnTw=
-github.com/xanzy/go-gitlab v0.84.0 h1:PdpCaskQSgcVDsx21c6ikf8Rfyo7SNtFAJwP9PrbCFE=
-github.com/xanzy/go-gitlab v0.84.0/go.mod h1:5ryv+MnpZStBH8I/77HuQBsMbBGANtVpLWC15qOjWAw=
+github.com/xanzy/go-gitlab v0.107.0 h1:P2CT9Uy9yN9lJo3FLxpMZ4xj6uWcpnigXsjvqJ6nd2Y=
+github.com/xanzy/go-gitlab v0.107.0/go.mod h1:wKNKh3GkYDMOsGmnfuX+ITCmDuSDWFO0G+C4AygL9RY=
github.com/xanzy/ssh-agent v0.3.3 h1:+/15pJfg/RsTxqYcX6fHqOXZwwMP+2VyYWJeWM2qQFM=
github.com/xanzy/ssh-agent v0.3.3/go.mod h1:6dzNDKs0J9rVPHPhaGCukekBHKqfl+L3KghI1Bc68Uw=
github.com/xeipuuv/gojsonpointer v0.0.0-20180127040702-4e3ac2762d5f/go.mod h1:N2zxlSyiKSe5eX1tZViRH5QA0qijqEDrYZiPEAiq3wU=
@@ -1291,8 +1289,8 @@ golang.org/x/oauth2 v0.0.0-20210514164344-f6687ab2804c/go.mod h1:KelEdhl1UZF7XfJ
golang.org/x/oauth2 v0.0.0-20220223155221-ee480838109b/go.mod h1:DAh4E804XQdzx2j+YRIaUnCqCV2RuMz24cGBJ5QYIrc=
golang.org/x/oauth2 v0.5.0/go.mod h1:9/XBHVqLaWO3/BRHs5jbpYCnOZVjj5V0ndyaAM7KB4I=
golang.org/x/oauth2 v0.6.0/go.mod h1:ycmewcwgD4Rpr3eZJLSB4Kyyljb3qDh40vJ8STE5HKw=
-golang.org/x/oauth2 v0.18.0 h1:09qnuIAgzdx1XplqJvW6CQqMCtGZykZWcXzPMPUusvI=
-golang.org/x/oauth2 v0.18.0/go.mod h1:Wf7knwG0MPoWIMMBgFlEaSUDaKskp0dCfrlJRJXbBi8=
+golang.org/x/oauth2 v0.22.0 h1:BzDx2FehcG7jJwgWLELCdmLuxk2i+x9UDpSiss2u0ZA=
+golang.org/x/oauth2 v0.22.0/go.mod h1:XYTD2NtWslqkgxebSiOHnXEap4TF09sJSc7H1sXbhtI=
golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
golang.org/x/sync v0.0.0-20181221193216-37e7f081c4d4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
@@ -1421,8 +1419,8 @@ golang.org/x/time v0.0.0-20180412165947-fbb02b2291d2/go.mod h1:tRJNPiyCQ0inRvYxb
golang.org/x/time v0.0.0-20181108054448-85acf8d2951c/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
golang.org/x/time v0.0.0-20190308202827-9d24e82272b4/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
golang.org/x/time v0.0.0-20191024005414-555d28b269f0/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
-golang.org/x/time v0.5.0 h1:o7cqy6amK/52YcAKIPlM3a+Fpj35zvRj2TP+e1xFSfk=
-golang.org/x/time v0.5.0/go.mod h1:3BpzKBy/shNhVucY/MWOyx10tF3SFh9QdLuxbVysPQM=
+golang.org/x/time v0.6.0 h1:eTDhh4ZXt5Qf0augr54TN6suAUudPcawVZeIAPU7D4U=
+golang.org/x/time v0.6.0/go.mod h1:3BpzKBy/shNhVucY/MWOyx10tF3SFh9QdLuxbVysPQM=
golang.org/x/tools v0.0.0-20180221164845-07fd8470d635/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
golang.org/x/tools v0.0.0-20180828015842-6cd1fcedba52/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
@@ -1505,8 +1503,6 @@ google.golang.org/appengine v1.6.1/go.mod h1:i06prIuMbXzDqacNJfV5OdTW448YApPu5ww
google.golang.org/appengine v1.6.5/go.mod h1:8WjMMxjGQR8xUklV/ARdw2HLXBOI7O7uCIDZVag1xfc=
google.golang.org/appengine v1.6.6/go.mod h1:8WjMMxjGQR8xUklV/ARdw2HLXBOI7O7uCIDZVag1xfc=
google.golang.org/appengine v1.6.7/go.mod h1:8WjMMxjGQR8xUklV/ARdw2HLXBOI7O7uCIDZVag1xfc=
-google.golang.org/appengine v1.6.8 h1:IhEN5q69dyKagZPYMSdIjS2HqprW324FRQZJcGqPAsM=
-google.golang.org/appengine v1.6.8/go.mod h1:1jJ3jBArFh5pcgW8gCtRJnepW8FzD1V44FJffLiz/Ds=
google.golang.org/genproto v0.0.0-20180817151627-c66870c02cf8/go.mod h1:JiN7NxoALGmiZfu7CAH4rXhgtRTLTxftemlI0sWmxmc=
google.golang.org/genproto v0.0.0-20190307195333-5fe7a883aa19/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE=
google.golang.org/genproto v0.0.0-20190418145605-e7d98fc518a7/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE=
@@ -1569,8 +1565,9 @@ google.golang.org/protobuf v1.26.0/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQ
google.golang.org/protobuf v1.28.0/go.mod h1:HV8QOd/L58Z+nl8r43ehVNZIU/HEI6OcFqwMG9pJV4I=
google.golang.org/protobuf v1.28.1/go.mod h1:HV8QOd/L58Z+nl8r43ehVNZIU/HEI6OcFqwMG9pJV4I=
google.golang.org/protobuf v1.30.0/go.mod h1:HV8QOd/L58Z+nl8r43ehVNZIU/HEI6OcFqwMG9pJV4I=
-google.golang.org/protobuf v1.33.0 h1:uNO2rsAINq/JlFpSdYEKIZ0uKD/R9cpdv0T+yoGwGmI=
google.golang.org/protobuf v1.33.0/go.mod h1:c6P6GXX6sHbq/GpV6MGZEdwhWPcYBgnhAHhKbcUYpos=
+google.golang.org/protobuf v1.34.2 h1:6xV6lTsCfpGD21XK49h7MhtcApnLqkfYgPcdHftf6hg=
+google.golang.org/protobuf v1.34.2/go.mod h1:qYOHts0dSfpeUzUFpOMr/WGzszTmLH+DiWniOlNbLDw=
gopkg.in/alecthomas/kingpin.v2 v2.2.6 h1:jMFz6MfLP0/4fUyZle81rXUoxOBFi19VUFKVDOQfozc=
gopkg.in/alecthomas/kingpin.v2 v2.2.6/go.mod h1:FMv+mEhP44yOT+4EoQTLFTRgOQ1FBLkstjWtayDeSgw=
gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
diff --git a/pkg/reporting/trackers/gitlab/gitlab.go b/pkg/reporting/trackers/gitlab/gitlab.go
index d816a26af6..fdb2b948f1 100644
--- a/pkg/reporting/trackers/gitlab/gitlab.go
+++ b/pkg/reporting/trackers/gitlab/gitlab.go
@@ -78,7 +78,7 @@ func (i *Integration) CreateIssue(event *output.ResultEvent) (*filters.CreateIss
if label := i.options.IssueLabel; label != "" {
labels = append(labels, label)
}
- customLabels := gitlab.Labels(labels)
+ customLabels := gitlab.LabelOptions(labels)
assigneeIDs := []int{i.userID}
if i.options.DuplicateIssueCheck {
searchIn := "title"
From d1f4c98cd76f82e556b4a9c9406c45bcec32ae7f Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Do=C4=9Fan=20Can=20Bak=C4=B1r?=
Date: Wed, 21 Aug 2024 15:03:41 +0300
Subject: [PATCH 164/283] Revert "remove redundant code"
This reverts commit 35a0d673ad8e12b11e90e8e0090feb26ea042b46.
---
pkg/protocols/common/protocolstate/state.go | 1 +
1 file changed, 1 insertion(+)
diff --git a/pkg/protocols/common/protocolstate/state.go b/pkg/protocols/common/protocolstate/state.go
index b21a1be9e5..b7c7796fa8 100644
--- a/pkg/protocols/common/protocolstate/state.go
+++ b/pkg/protocols/common/protocolstate/state.go
@@ -214,5 +214,6 @@ func Close() {
Dialer.Close()
Dialer = nil
}
+ Dialer = nil
StopActiveMemGuardian()
}
From 5e102b782ba28386288460f7e6f1fec08ea514fa Mon Sep 17 00:00:00 2001
From: mzack9999
Date: Wed, 21 Aug 2024 16:09:47 +0200
Subject: [PATCH 165/283] fixing race + nil crash
---
pkg/protocols/common/protocolstate/state.go | 15 +++++++++++++--
pkg/protocols/http/httpclientpool/clientpool.go | 4 ++--
pkg/protocols/http/request.go | 6 +++++-
3 files changed, 20 insertions(+), 5 deletions(-)
diff --git a/pkg/protocols/common/protocolstate/state.go b/pkg/protocols/common/protocolstate/state.go
index b7c7796fa8..5c56fdf77e 100644
--- a/pkg/protocols/common/protocolstate/state.go
+++ b/pkg/protocols/common/protocolstate/state.go
@@ -5,6 +5,7 @@ import (
"fmt"
"net"
"net/url"
+ "sync"
"github.com/go-sql-driver/mysql"
"github.com/pkg/errors"
@@ -19,9 +20,17 @@ import (
// Dialer is a shared fastdialer instance for host DNS resolution
var (
- Dialer *fastdialer.Dialer
+ muDialer sync.RWMutex
+ Dialer *fastdialer.Dialer
)
+func GetDialer() *fastdialer.Dialer {
+ muDialer.RLock()
+ defer muDialer.RUnlock()
+
+ return Dialer
+}
+
func ShouldInit() bool {
return Dialer == nil
}
@@ -210,10 +219,12 @@ func interfaceAddresses(interfaceName string) ([]net.Addr, error) {
// Close closes the global shared fastdialer
func Close() {
+ muDialer.Lock()
+ defer muDialer.Unlock()
+
if Dialer != nil {
Dialer.Close()
Dialer = nil
}
- Dialer = nil
StopActiveMemGuardian()
}
diff --git a/pkg/protocols/http/httpclientpool/clientpool.go b/pkg/protocols/http/httpclientpool/clientpool.go
index 5750d5aead..65879818f6 100644
--- a/pkg/protocols/http/httpclientpool/clientpool.go
+++ b/pkg/protocols/http/httpclientpool/clientpool.go
@@ -250,7 +250,7 @@ func wrappedGet(options *types.Options, configuration *Configuration) (*retryabl
transport := &http.Transport{
ForceAttemptHTTP2: options.ForceAttemptHTTP2,
- DialContext: protocolstate.Dialer.Dial,
+ DialContext: protocolstate.GetDialer().Dial,
DialTLSContext: func(ctx context.Context, network, addr string) (net.Conn, error) {
if options.TlsImpersonate {
return protocolstate.Dialer.DialTLSWithConfigImpersonate(ctx, network, addr, tlsConfig, impersonate.Random, nil)
@@ -258,7 +258,7 @@ func wrappedGet(options *types.Options, configuration *Configuration) (*retryabl
if options.HasClientCertificates() || options.ForceAttemptHTTP2 {
return protocolstate.Dialer.DialTLSWithConfig(ctx, network, addr, tlsConfig)
}
- return protocolstate.Dialer.DialTLS(ctx, network, addr)
+ return protocolstate.GetDialer().DialTLS(ctx, network, addr)
},
MaxIdleConns: maxIdleConns,
MaxIdleConnsPerHost: maxIdleConnsPerHost,
diff --git a/pkg/protocols/http/request.go b/pkg/protocols/http/request.go
index 3955325599..5a1219b83f 100644
--- a/pkg/protocols/http/request.go
+++ b/pkg/protocols/http/request.go
@@ -942,7 +942,11 @@ func (request *Request) executeRequest(input *contextargs.Context, generatedRequ
if input.MetaInput.CustomIP != "" {
outputEvent["ip"] = input.MetaInput.CustomIP
} else {
- outputEvent["ip"] = protocolstate.Dialer.GetDialedIP(hostname)
+ dialer := protocolstate.GetDialer()
+ if dialer != nil {
+ outputEvent["ip"] = dialer.GetDialedIP(hostname)
+ }
+
// try getting cname
request.addCNameIfAvailable(hostname, outputEvent)
}
From ddf9b2fbec27224a536e38ee0ecde5874f7b2c5b Mon Sep 17 00:00:00 2001
From: mzack9999
Date: Wed, 21 Aug 2024 16:29:00 +0200
Subject: [PATCH 166/283] fix
---
lib/sdk.go | 3 ++-
1 file changed, 2 insertions(+), 1 deletion(-)
diff --git a/lib/sdk.go b/lib/sdk.go
index 04b084f244..daeb68c14b 100644
--- a/lib/sdk.go
+++ b/lib/sdk.go
@@ -178,7 +178,8 @@ func (e *NucleiEngine) SignTemplate(tmplSigner *signer.TemplateSigner, data []by
if err != nil {
return data, err
}
- buff := bytes.NewBuffer(signer.RemoveSignatureFromData(data))
+ _, content := signer.ExtractSignatureAndContent(data)
+ buff := bytes.NewBuffer(content)
buff.WriteString("\n" + signatureData)
return buff.Bytes(), err
}
From bfdc507c6cef6db636ae8b3e693002271ed4ddab Mon Sep 17 00:00:00 2001
From: mzack9999
Date: Wed, 21 Aug 2024 16:34:53 +0200
Subject: [PATCH 167/283] lint
---
pkg/external/customtemplates/gitlab.go | 6 +++---
1 file changed, 3 insertions(+), 3 deletions(-)
diff --git a/pkg/external/customtemplates/gitlab.go b/pkg/external/customtemplates/gitlab.go
index 40f191423a..fbb9a61cef 100644
--- a/pkg/external/customtemplates/gitlab.go
+++ b/pkg/external/customtemplates/gitlab.go
@@ -83,8 +83,8 @@ func (bk *customTemplateGitLabRepo) Download(_ context.Context) {
// Get the directory listing for the files in the project
tree, _, err := bk.gitLabClient.Repositories.ListTree(projectID, &gitlab.ListTreeOptions{
- Ref: gitlab.String(project.DefaultBranch),
- Recursive: gitlab.Bool(true),
+ Ref: gitlab.Ptr(project.DefaultBranch),
+ Recursive: gitlab.Ptr(true),
})
if err != nil {
gologger.Error().Msgf("error retrieving files from GitLab project: %s (%d) %s", project.Name, projectID, err)
@@ -95,7 +95,7 @@ func (bk *customTemplateGitLabRepo) Download(_ context.Context) {
// If the object is not a file or file extension is not .yaml, skip it
if file.Type == "blob" && filepath.Ext(file.Path) == ".yaml" {
gf := &gitlab.GetFileOptions{
- Ref: gitlab.String(project.DefaultBranch),
+ Ref: gitlab.Ptr(project.DefaultBranch),
}
f, _, err := bk.gitLabClient.RepositoryFiles.GetFile(projectID, file.Path, gf)
if err != nil {
From 56af26a502692774dc879f5fe5f10d3caf5f0e6e Mon Sep 17 00:00:00 2001
From: Dogan Can Bakir <65292895+dogancanbakir@users.noreply.github.com>
Date: Thu, 22 Aug 2024 15:49:07 +0300
Subject: [PATCH 168/283] Add auto assign (#5555)
* add auto assign
* commit to last commit
---
.github/auto_assign.yml | 8 ++++++++
1 file changed, 8 insertions(+)
create mode 100644 .github/auto_assign.yml
diff --git a/.github/auto_assign.yml b/.github/auto_assign.yml
new file mode 100644
index 0000000000..127400e160
--- /dev/null
+++ b/.github/auto_assign.yml
@@ -0,0 +1,8 @@
+addReviewers: true
+reviewers:
+ - dogancanbakir
+
+numberOfReviewers: 1
+# A list of keywords to be skipped the process that add reviewers if pull requests include it
+# skipKeywords:
+# - wip
\ No newline at end of file
From 710f1ad240a1fedbdd0e91573c633f218e9fb409 Mon Sep 17 00:00:00 2001
From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com>
Date: Mon, 26 Aug 2024 05:02:29 +0000
Subject: [PATCH 169/283] chore(deps): bump
github.com/projectdiscovery/gologger
Bumps [github.com/projectdiscovery/gologger](https://github.com/projectdiscovery/gologger) from 1.1.19 to 1.1.21.
- [Release notes](https://github.com/projectdiscovery/gologger/releases)
- [Commits](https://github.com/projectdiscovery/gologger/compare/v1.1.19...v1.1.21)
---
updated-dependencies:
- dependency-name: github.com/projectdiscovery/gologger
dependency-type: direct:production
update-type: version-update:semver-patch
...
Signed-off-by: dependabot[bot]
---
go.mod | 4 ++--
go.sum | 8 ++++----
2 files changed, 6 insertions(+), 6 deletions(-)
diff --git a/go.mod b/go.mod
index 026a96169e..05ddf348f1 100644
--- a/go.mod
+++ b/go.mod
@@ -84,7 +84,7 @@ require (
github.com/projectdiscovery/fasttemplate v0.0.2
github.com/projectdiscovery/go-smb2 v0.0.0-20240129202741-052cc450c6cb
github.com/projectdiscovery/goflags v0.1.63
- github.com/projectdiscovery/gologger v1.1.19
+ github.com/projectdiscovery/gologger v1.1.21
github.com/projectdiscovery/gostruct v0.0.2
github.com/projectdiscovery/gozero v0.0.2
github.com/projectdiscovery/httpx v1.6.7
@@ -96,7 +96,7 @@ require (
github.com/projectdiscovery/tlsx v1.1.6
github.com/projectdiscovery/uncover v1.0.9
github.com/projectdiscovery/useragent v0.0.65
- github.com/projectdiscovery/utils v0.2.5
+ github.com/projectdiscovery/utils v0.2.6
github.com/projectdiscovery/wappalyzergo v0.1.14
github.com/redis/go-redis/v9 v9.1.0
github.com/seh-msft/burpxml v1.0.1
diff --git a/go.sum b/go.sum
index 22a4618330..a5d836f324 100644
--- a/go.sum
+++ b/go.sum
@@ -842,8 +842,8 @@ github.com/projectdiscovery/go-smb2 v0.0.0-20240129202741-052cc450c6cb h1:rutG90
github.com/projectdiscovery/go-smb2 v0.0.0-20240129202741-052cc450c6cb/go.mod h1:FLjF1DmZ+POoGEiIQdWuYVwS++C/GwpX8YaCsTSm1RY=
github.com/projectdiscovery/goflags v0.1.63 h1:xy3nqpzmSFEmG0LKTv3xLcp9LMbvbFkLZcjcnFe3xR4=
github.com/projectdiscovery/goflags v0.1.63/go.mod h1:ptFbCm+5RmSHTSG4oniwBRRHJlguBTOOfMUfezeGbZ0=
-github.com/projectdiscovery/gologger v1.1.19 h1:b7cU32XuDrDiwhr7hlDeE6mfj/nENBtHEohe51txJCE=
-github.com/projectdiscovery/gologger v1.1.19/go.mod h1:DbeKwx9IEfcvnclImX5gBlhIKUuOZwOM5itdpYXl+54=
+github.com/projectdiscovery/gologger v1.1.21 h1:FD9MGE9adY/y2IxDpQgPZIvZAkQIx3bvcuY+d1FLQUo=
+github.com/projectdiscovery/gologger v1.1.21/go.mod h1:eyvlcHiSuzJQtwmh+fcfwZVaqVgSGtEP1oL1gJFgP3c=
github.com/projectdiscovery/gostruct v0.0.2 h1:s8gP8ApugGM4go1pA+sVlPDXaWqNP5BBDDSv7VEdG1M=
github.com/projectdiscovery/gostruct v0.0.2/go.mod h1:H86peL4HKwMXcQQtEa6lmC8FuD9XFt6gkNR0B/Mu5PE=
github.com/projectdiscovery/gozero v0.0.2 h1:8fJeaCjxL9tpm33uG/RsCQs6HGM/NE6eA3cjkilRQ+E=
@@ -884,8 +884,8 @@ github.com/projectdiscovery/uncover v1.0.9 h1:s5RbkD/V4r8QcPkys4gTTqMuRSgXq0Jpre
github.com/projectdiscovery/uncover v1.0.9/go.mod h1:2PUF3SpB5QNIJ8epaB2xbRzkPaxEAWRDm3Ir2ijt81U=
github.com/projectdiscovery/useragent v0.0.65 h1:x78ZwWdqpzokOHxLITUXvq+ljkTKc19z3ILGtoV1N70=
github.com/projectdiscovery/useragent v0.0.65/go.mod h1:deOP8YLJU6SCzM8k+K8PjkcOF4Ux0spqyO4ODZGIT4A=
-github.com/projectdiscovery/utils v0.2.5 h1:puiw3/gK1WTEj9ZKnW00tAS0ex+ETqbLJ8juRuejA98=
-github.com/projectdiscovery/utils v0.2.5/go.mod h1:fv3WC6m/+bVT+wH+fMB+JsKviEC/UAuDSeKxWPlK3kk=
+github.com/projectdiscovery/utils v0.2.6 h1:7ZZ+tK8u3vO/X7i0HXwybNusFBtYnM363bxDxJkVnso=
+github.com/projectdiscovery/utils v0.2.6/go.mod h1:XFy8Ldt537TGkj3snH0zXfF9LigRfmHPGCNfDolSdEQ=
github.com/projectdiscovery/wappalyzergo v0.1.14 h1:nt1IM4RUmqeymsXk4h6BsZbKDoS2hjFvPkT2GaI1rz4=
github.com/projectdiscovery/wappalyzergo v0.1.14/go.mod h1:/hzgxkBFTMe2wDbA93nFfoMjULw7/vIZ9QPSAnCgUa8=
github.com/projectdiscovery/yamldoc-go v1.0.4 h1:eZoESapnMw6WAHiVgRwNqvbJEfNHEH148uthhFbG5jE=
From 4a39fdb8d6553793feb9c7310d9ee348ff927a1b Mon Sep 17 00:00:00 2001
From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com>
Date: Mon, 26 Aug 2024 05:26:22 +0000
Subject: [PATCH 170/283] chore(deps): bump github.com/projectdiscovery/utils
from 0.2.5 to 0.2.7
Bumps [github.com/projectdiscovery/utils](https://github.com/projectdiscovery/utils) from 0.2.5 to 0.2.7.
- [Release notes](https://github.com/projectdiscovery/utils/releases)
- [Changelog](https://github.com/projectdiscovery/utils/blob/main/CHANGELOG.md)
- [Commits](https://github.com/projectdiscovery/utils/compare/v0.2.5...v0.2.7)
---
updated-dependencies:
- dependency-name: github.com/projectdiscovery/utils
dependency-type: direct:production
update-type: version-update:semver-patch
...
Signed-off-by: dependabot[bot]
---
go.mod | 2 +-
go.sum | 4 ++--
2 files changed, 3 insertions(+), 3 deletions(-)
diff --git a/go.mod b/go.mod
index 05ddf348f1..e7c5efd226 100644
--- a/go.mod
+++ b/go.mod
@@ -96,7 +96,7 @@ require (
github.com/projectdiscovery/tlsx v1.1.6
github.com/projectdiscovery/uncover v1.0.9
github.com/projectdiscovery/useragent v0.0.65
- github.com/projectdiscovery/utils v0.2.6
+ github.com/projectdiscovery/utils v0.2.7
github.com/projectdiscovery/wappalyzergo v0.1.14
github.com/redis/go-redis/v9 v9.1.0
github.com/seh-msft/burpxml v1.0.1
diff --git a/go.sum b/go.sum
index a5d836f324..28af644eed 100644
--- a/go.sum
+++ b/go.sum
@@ -884,8 +884,8 @@ github.com/projectdiscovery/uncover v1.0.9 h1:s5RbkD/V4r8QcPkys4gTTqMuRSgXq0Jpre
github.com/projectdiscovery/uncover v1.0.9/go.mod h1:2PUF3SpB5QNIJ8epaB2xbRzkPaxEAWRDm3Ir2ijt81U=
github.com/projectdiscovery/useragent v0.0.65 h1:x78ZwWdqpzokOHxLITUXvq+ljkTKc19z3ILGtoV1N70=
github.com/projectdiscovery/useragent v0.0.65/go.mod h1:deOP8YLJU6SCzM8k+K8PjkcOF4Ux0spqyO4ODZGIT4A=
-github.com/projectdiscovery/utils v0.2.6 h1:7ZZ+tK8u3vO/X7i0HXwybNusFBtYnM363bxDxJkVnso=
-github.com/projectdiscovery/utils v0.2.6/go.mod h1:XFy8Ldt537TGkj3snH0zXfF9LigRfmHPGCNfDolSdEQ=
+github.com/projectdiscovery/utils v0.2.7 h1:XWdz7SscL++jqsnQ9ecHzSZE0RK33tyPcnqcXw+vmKs=
+github.com/projectdiscovery/utils v0.2.7/go.mod h1:N0N7tbdNFPegd9NpJ3onCPClaBrERcOIB88yww6UCF8=
github.com/projectdiscovery/wappalyzergo v0.1.14 h1:nt1IM4RUmqeymsXk4h6BsZbKDoS2hjFvPkT2GaI1rz4=
github.com/projectdiscovery/wappalyzergo v0.1.14/go.mod h1:/hzgxkBFTMe2wDbA93nFfoMjULw7/vIZ9QPSAnCgUa8=
github.com/projectdiscovery/yamldoc-go v1.0.4 h1:eZoESapnMw6WAHiVgRwNqvbJEfNHEH148uthhFbG5jE=
From 253fbc36190aec186bdf8c32e1fea34350ad8d8e Mon Sep 17 00:00:00 2001
From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com>
Date: Mon, 26 Aug 2024 05:26:23 +0000
Subject: [PATCH 171/283] chore(deps): bump github.com/projectdiscovery/goflags
Bumps [github.com/projectdiscovery/goflags](https://github.com/projectdiscovery/goflags) from 0.1.63 to 0.1.64.
- [Release notes](https://github.com/projectdiscovery/goflags/releases)
- [Commits](https://github.com/projectdiscovery/goflags/compare/v0.1.63...v0.1.64)
---
updated-dependencies:
- dependency-name: github.com/projectdiscovery/goflags
dependency-type: direct:production
update-type: version-update:semver-patch
...
Signed-off-by: dependabot[bot]
---
go.mod | 2 +-
go.sum | 4 ++--
2 files changed, 3 insertions(+), 3 deletions(-)
diff --git a/go.mod b/go.mod
index 05ddf348f1..851fc631ee 100644
--- a/go.mod
+++ b/go.mod
@@ -83,7 +83,7 @@ require (
github.com/projectdiscovery/dsl v0.1.8
github.com/projectdiscovery/fasttemplate v0.0.2
github.com/projectdiscovery/go-smb2 v0.0.0-20240129202741-052cc450c6cb
- github.com/projectdiscovery/goflags v0.1.63
+ github.com/projectdiscovery/goflags v0.1.64
github.com/projectdiscovery/gologger v1.1.21
github.com/projectdiscovery/gostruct v0.0.2
github.com/projectdiscovery/gozero v0.0.2
diff --git a/go.sum b/go.sum
index a5d836f324..fe4f540f3d 100644
--- a/go.sum
+++ b/go.sum
@@ -840,8 +840,8 @@ github.com/projectdiscovery/freeport v0.0.5 h1:jnd3Oqsl4S8n0KuFkE5Hm8WGDP24ITBvm
github.com/projectdiscovery/freeport v0.0.5/go.mod h1:PY0bxSJ34HVy67LHIeF3uIutiCSDwOqKD8ruBkdiCwE=
github.com/projectdiscovery/go-smb2 v0.0.0-20240129202741-052cc450c6cb h1:rutG906Drtbpz4DwU5mhGIeOhRcktDH4cGQitGUMAsg=
github.com/projectdiscovery/go-smb2 v0.0.0-20240129202741-052cc450c6cb/go.mod h1:FLjF1DmZ+POoGEiIQdWuYVwS++C/GwpX8YaCsTSm1RY=
-github.com/projectdiscovery/goflags v0.1.63 h1:xy3nqpzmSFEmG0LKTv3xLcp9LMbvbFkLZcjcnFe3xR4=
-github.com/projectdiscovery/goflags v0.1.63/go.mod h1:ptFbCm+5RmSHTSG4oniwBRRHJlguBTOOfMUfezeGbZ0=
+github.com/projectdiscovery/goflags v0.1.64 h1:FDfwdt9N97Hi8OuhbkDlKtVttpc/CRMIWQVa08VsHsI=
+github.com/projectdiscovery/goflags v0.1.64/go.mod h1:3FyHIVQtnycNOc1LE3O1jj/XR5XuMdF9QfHd0ujhnX4=
github.com/projectdiscovery/gologger v1.1.21 h1:FD9MGE9adY/y2IxDpQgPZIvZAkQIx3bvcuY+d1FLQUo=
github.com/projectdiscovery/gologger v1.1.21/go.mod h1:eyvlcHiSuzJQtwmh+fcfwZVaqVgSGtEP1oL1gJFgP3c=
github.com/projectdiscovery/gostruct v0.0.2 h1:s8gP8ApugGM4go1pA+sVlPDXaWqNP5BBDDSv7VEdG1M=
From 2670b6865f6590cb4fee720bcaf1a51b90fe256f Mon Sep 17 00:00:00 2001
From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com>
Date: Mon, 26 Aug 2024 05:53:48 +0000
Subject: [PATCH 172/283] chore(deps): bump github.com/projectdiscovery/tlsx
from 1.1.6 to 1.1.7
Bumps [github.com/projectdiscovery/tlsx](https://github.com/projectdiscovery/tlsx) from 1.1.6 to 1.1.7.
- [Release notes](https://github.com/projectdiscovery/tlsx/releases)
- [Changelog](https://github.com/projectdiscovery/tlsx/blob/main/.goreleaser.yml)
- [Commits](https://github.com/projectdiscovery/tlsx/compare/v1.1.6...v1.1.7)
---
updated-dependencies:
- dependency-name: github.com/projectdiscovery/tlsx
dependency-type: direct:production
update-type: version-update:semver-patch
...
Signed-off-by: dependabot[bot]
---
go.mod | 2 +-
go.sum | 4 ++--
2 files changed, 3 insertions(+), 3 deletions(-)
diff --git a/go.mod b/go.mod
index e7c5efd226..4832777901 100644
--- a/go.mod
+++ b/go.mod
@@ -93,7 +93,7 @@ require (
github.com/projectdiscovery/ratelimit v0.0.49
github.com/projectdiscovery/rdap v0.9.1-0.20221108103045-9865884d1917
github.com/projectdiscovery/sarif v0.0.1
- github.com/projectdiscovery/tlsx v1.1.6
+ github.com/projectdiscovery/tlsx v1.1.7
github.com/projectdiscovery/uncover v1.0.9
github.com/projectdiscovery/useragent v0.0.65
github.com/projectdiscovery/utils v0.2.7
diff --git a/go.sum b/go.sum
index 28af644eed..35385e3eb6 100644
--- a/go.sum
+++ b/go.sum
@@ -878,8 +878,8 @@ github.com/projectdiscovery/sarif v0.0.1 h1:C2Tyj0SGOKbCLgHrx83vaE6YkzXEVrMXYRGL
github.com/projectdiscovery/sarif v0.0.1/go.mod h1:cEYlDu8amcPf6b9dSakcz2nNnJsoz4aR6peERwV+wuQ=
github.com/projectdiscovery/stringsutil v0.0.2 h1:uzmw3IVLJSMW1kEg8eCStG/cGbYYZAja8BH3LqqJXMA=
github.com/projectdiscovery/stringsutil v0.0.2/go.mod h1:EJ3w6bC5fBYjVou6ryzodQq37D5c6qbAYQpGmAy+DC0=
-github.com/projectdiscovery/tlsx v1.1.6 h1:iw2zwKbd2+kRQ8J1G4dLmS0CLyemd/tKz1UzcNsC77A=
-github.com/projectdiscovery/tlsx v1.1.6/go.mod h1:s7SRRFdrwIZBK/RXXZi4CR/CubqFSvp8h5Bk1srEZIo=
+github.com/projectdiscovery/tlsx v1.1.7 h1:eSsl/SmTDL/z2CMeSrbssk4f/9oOotMP1SgXl3yynSM=
+github.com/projectdiscovery/tlsx v1.1.7/go.mod h1:g66QQ4/y4tLVjoGbzWIv+Q6xwFzxfJbEDx86Y1dYHDM=
github.com/projectdiscovery/uncover v1.0.9 h1:s5RbkD/V4r8QcPkys4gTTqMuRSgXq0JprejqLSopN9Y=
github.com/projectdiscovery/uncover v1.0.9/go.mod h1:2PUF3SpB5QNIJ8epaB2xbRzkPaxEAWRDm3Ir2ijt81U=
github.com/projectdiscovery/useragent v0.0.65 h1:x78ZwWdqpzokOHxLITUXvq+ljkTKc19z3ILGtoV1N70=
From 4a85e73fbb717109f98dd3e8f96bc524b8cda26b Mon Sep 17 00:00:00 2001
From: Dogan Can Bakir <65292895+dogancanbakir@users.noreply.github.com>
Date: Mon, 26 Aug 2024 19:48:45 +0300
Subject: [PATCH 173/283] skip dependabot prs (#5575)
---
.github/auto_assign.yml | 5 ++---
1 file changed, 2 insertions(+), 3 deletions(-)
diff --git a/.github/auto_assign.yml b/.github/auto_assign.yml
index 127400e160..0c65e536ca 100644
--- a/.github/auto_assign.yml
+++ b/.github/auto_assign.yml
@@ -3,6 +3,5 @@ reviewers:
- dogancanbakir
numberOfReviewers: 1
-# A list of keywords to be skipped the process that add reviewers if pull requests include it
-# skipKeywords:
-# - wip
\ No newline at end of file
+skipKeywords:
+ - '@dependabot'
\ No newline at end of file
From aac1af1308da2187d1f6bb9aa73c8fe226c4fe6b Mon Sep 17 00:00:00 2001
From: Dwi Siswanto
Date: Wed, 28 Aug 2024 14:11:02 +0700
Subject: [PATCH 174/283] refactor(fuzz): use `mapsutil.Map` type (#5558)
* refactor(fuzz): use `mapsutil.Map` type
Signed-off-by: Dwi Siswanto
* fix(headless): handle empty `key` in `*Value.SetParsedValue`
Signed-off-by: Dwi Siswanto
* feat(fuzz): add type assertion checks
Signed-off-by: Dwi Siswanto
---------
Signed-off-by: Dwi Siswanto
---
pkg/fuzz/component/path.go | 21 ++++++++++++++++++---
pkg/fuzz/component/value.go | 12 +++++++++++-
pkg/fuzz/dataformat/kv.go | 2 +-
3 files changed, 30 insertions(+), 5 deletions(-)
diff --git a/pkg/fuzz/component/path.go b/pkg/fuzz/component/path.go
index b9aebd61af..a81955167a 100644
--- a/pkg/fuzz/component/path.go
+++ b/pkg/fuzz/component/path.go
@@ -7,6 +7,7 @@ import (
"github.com/projectdiscovery/nuclei/v3/pkg/fuzz/dataformat"
"github.com/projectdiscovery/retryablehttp-go"
+ "github.com/projectdiscovery/utils/maps"
urlutil "github.com/projectdiscovery/utils/url"
)
@@ -81,7 +82,7 @@ func (q *Path) Delete(key string) error {
// Rebuild returns a new request with the
// component rebuilt
func (q *Path) Rebuild() (*retryablehttp.Request, error) {
- originalValues := make(map[string]interface{})
+ originalValues := mapsutil.Map[string, any]{}
splitted := strings.Split(q.req.URL.Path, "/")
for i := range splitted {
pathTillNow := strings.Join(splitted[:i+1], "/")
@@ -95,8 +96,22 @@ func (q *Path) Rebuild() (*retryablehttp.Request, error) {
lengthSplitted := len(q.value.parsed.Map)
for i := lengthSplitted; i > 0; i-- {
key := strconv.Itoa(i)
- original := originalValues[key].(string)
- new := q.value.parsed.Map[key].(string)
+
+ original, ok := originalValues.GetOrDefault(key, "").(string)
+ if !ok {
+ continue
+ }
+
+ new, ok := q.value.parsed.Map.GetOrDefault(key, "").(string)
+ if !ok {
+ continue
+ }
+
+ if new == original {
+ // no need to replace
+ continue
+ }
+
originalPath = strings.Replace(originalPath, original, new, 1)
}
diff --git a/pkg/fuzz/component/value.go b/pkg/fuzz/component/value.go
index ad2044cf1b..030c5194bf 100644
--- a/pkg/fuzz/component/value.go
+++ b/pkg/fuzz/component/value.go
@@ -73,12 +73,22 @@ func (v *Value) SetParsed(data dataformat.KV, dataFormat string) {
// SetParsedValue sets the parsed value for a key
// in the parsed map
-func (v *Value) SetParsedValue(key string, value string) bool {
+func (v *Value) SetParsedValue(key, value string) bool {
+ if key == "" {
+ return false
+ }
+
origValue := v.parsed.Get(key)
if origValue == nil {
v.parsed.Set(key, value)
return true
}
+
+ // TODO(dwisiswant0): I'm sure that this can be simplified because
+ // `dataformat.KV.*` is a type of `mapsutil.*` where the value is `any`. So,
+ // it looks like we won't type conversion here or even have its own methods
+ // inside `dataformat.KV`.
+
// If the value is a list, append to it
// otherwise replace it
switch v := origValue.(type) {
diff --git a/pkg/fuzz/dataformat/kv.go b/pkg/fuzz/dataformat/kv.go
index 72bd0da62e..3036a1c76c 100644
--- a/pkg/fuzz/dataformat/kv.go
+++ b/pkg/fuzz/dataformat/kv.go
@@ -13,7 +13,7 @@ import (
// if it's not important/significant (ex: json,xml) we use map
// this also allows us to iteratively implement ordered map
type KV struct {
- Map map[string]interface{}
+ Map mapsutil.Map[string, any]
OrderedMap *mapsutil.OrderedMap[string, any]
}
From bf58b4d756ce8e2ba595024c54e87b4cdc2ad1c3 Mon Sep 17 00:00:00 2001
From: Peter Dave Hello
Date: Wed, 28 Aug 2024 18:54:15 +0800
Subject: [PATCH 175/283] Remove conflicting `-U` option for `apk` in
Dockerfile (#5578)
The `--no-cache` option updates the index and avoids temporary files, making `-U` unnecessary, consistent with the subsequent `apk add`.
---
Dockerfile | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/Dockerfile b/Dockerfile
index 9dde3d7e27..45f92d4485 100644
--- a/Dockerfile
+++ b/Dockerfile
@@ -9,7 +9,7 @@ RUN go build ./cmd/nuclei
# Release
FROM alpine:3.18.6
-RUN apk -U upgrade --no-cache \
+RUN apk upgrade --no-cache \
&& apk add --no-cache bind-tools chromium ca-certificates
COPY --from=build-env /app/nuclei /usr/local/bin/
From 6b71af448a995d76d3069aa1326f1c85946c346b Mon Sep 17 00:00:00 2001
From: Dogan Can Bakir <65292895+dogancanbakir@users.noreply.github.com>
Date: Wed, 28 Aug 2024 13:57:43 +0300
Subject: [PATCH 176/283] Fixed issue with `-ms` option to scan non accessible
host (#5576)
* fail if OnResult callback is not called
* generate error message from error logs
* try..parse..
* fix lint
* add error message to last matcher event
* fix network protocol error logging
* log returned log from ExecuteWithResults
* add back specific logging
* clean up the msg
* minor
* init integration test for -ms
* add tests for http,network,js,ws protocols
* fix lint
* fix network test
* return err for dns protocol
* add integration test for dns protocol
---
cmd/integration-test/integration-test.go | 1 +
cmd/integration-test/matcher-status.go | 119 +++++++++++++++++++++++
pkg/protocols/dns/request.go | 4 +-
pkg/protocols/network/request.go | 4 +-
pkg/protocols/network/request_test.go | 2 +-
pkg/scan/scan_context.go | 6 +-
pkg/tmplexec/exec.go | 57 +++++++++++
7 files changed, 187 insertions(+), 6 deletions(-)
create mode 100644 cmd/integration-test/matcher-status.go
diff --git a/cmd/integration-test/integration-test.go b/cmd/integration-test/integration-test.go
index 180efcfc6e..84ec6790f8 100644
--- a/cmd/integration-test/integration-test.go
+++ b/cmd/integration-test/integration-test.go
@@ -55,6 +55,7 @@ var (
"dsl": dslTestcases,
"flow": flowTestcases,
"javascript": jsTestcases,
+ "matcher-status": matcherStatusTestcases,
}
// flakyTests are run with a retry count of 3
flakyTests = map[string]bool{
diff --git a/cmd/integration-test/matcher-status.go b/cmd/integration-test/matcher-status.go
new file mode 100644
index 0000000000..b88763720a
--- /dev/null
+++ b/cmd/integration-test/matcher-status.go
@@ -0,0 +1,119 @@
+package main
+
+import (
+ "encoding/json"
+ "fmt"
+
+ "github.com/projectdiscovery/nuclei/v3/pkg/output"
+ "github.com/projectdiscovery/nuclei/v3/pkg/testutils"
+)
+
+var matcherStatusTestcases = []TestCaseInfo{
+ {Path: "protocols/http/get.yaml", TestCase: &httpNoAccess{}},
+ {Path: "protocols/network/net-https.yaml", TestCase: &networkNoAccess{}},
+ {Path: "protocols/headless/headless-basic.yaml", TestCase: &headlessNoAccess{}},
+ {Path: "protocols/javascript/net-https.yaml", TestCase: &javascriptNoAccess{}},
+ {Path: "protocols/websocket/basic.yaml", TestCase: &websocketNoAccess{}},
+ {Path: "protocols/dns/a.yaml", TestCase: &dnsNoAccess{}},
+}
+
+type httpNoAccess struct{}
+
+func (h *httpNoAccess) Execute(filePath string) error {
+ results, err := testutils.RunNucleiTemplateAndGetResults(filePath, "trust_me_bro.real", debug, "-ms", "-j")
+ if err != nil {
+ return err
+ }
+ event := &output.ResultEvent{}
+ _ = json.Unmarshal([]byte(results[0]), event)
+
+ if event.Error != "no address found for host" {
+ return fmt.Errorf("unexpected result: expecting \"no address found for host\" error but got none")
+ }
+ return nil
+}
+
+type networkNoAccess struct{}
+
+// Execute executes a test case and returns an error if occurred
+func (h *networkNoAccess) Execute(filePath string) error {
+ results, err := testutils.RunNucleiTemplateAndGetResults(filePath, "trust_me_bro.real", debug, "-ms", "-j")
+ if err != nil {
+ return err
+ }
+ event := &output.ResultEvent{}
+ _ = json.Unmarshal([]byte(results[0]), event)
+
+ if event.Error != "no address found for host" {
+ return fmt.Errorf("unexpected result: expecting \"no address found for host\" error but got \"%s\"", event.Error)
+ }
+ return nil
+}
+
+type headlessNoAccess struct{}
+
+// Execute executes a test case and returns an error if occurred
+func (h *headlessNoAccess) Execute(filePath string) error {
+ results, err := testutils.RunNucleiTemplateAndGetResults(filePath, "trust_me_bro.real", debug, "-headless", "-ms", "-j")
+ if err != nil {
+ return err
+ }
+ event := &output.ResultEvent{}
+ _ = json.Unmarshal([]byte(results[0]), event)
+
+ if event.Error == "" {
+ return fmt.Errorf("unexpected result: expecting an error but got \"%s\"", event.Error)
+ }
+ return nil
+}
+
+type javascriptNoAccess struct{}
+
+// Execute executes a test case and returns an error if occurred
+func (h *javascriptNoAccess) Execute(filePath string) error {
+ results, err := testutils.RunNucleiTemplateAndGetResults(filePath, "trust_me_bro.real", debug, "-ms", "-j")
+ if err != nil {
+ return err
+ }
+ event := &output.ResultEvent{}
+ _ = json.Unmarshal([]byte(results[0]), event)
+
+ if event.Error == "" {
+ return fmt.Errorf("unexpected result: expecting an error but got \"%s\"", event.Error)
+ }
+ return nil
+}
+
+type websocketNoAccess struct{}
+
+// Execute executes a test case and returns an error if occurred
+func (h *websocketNoAccess) Execute(filePath string) error {
+ results, err := testutils.RunNucleiTemplateAndGetResults(filePath, "ws://trust_me_bro.real", debug, "-ms", "-j")
+ if err != nil {
+ return err
+ }
+ event := &output.ResultEvent{}
+ _ = json.Unmarshal([]byte(results[0]), event)
+
+ if event.Error == "" {
+ return fmt.Errorf("unexpected result: expecting an error but got \"%s\"", event.Error)
+ }
+ return nil
+}
+
+type dnsNoAccess struct{}
+
+// Execute executes a test case and returns an error if occurred
+func (h *dnsNoAccess) Execute(filePath string) error {
+ results, err := testutils.RunNucleiTemplateAndGetResults(filePath, "trust_me_bro.real", debug, "-ms", "-j")
+ if err != nil {
+ return err
+ }
+ event := &output.ResultEvent{}
+ _ = json.Unmarshal([]byte(results[0]), event)
+
+ if event.Error == "" {
+ return fmt.Errorf("unexpected result: expecting an error but got \"%s\"", event.Error)
+ }
+ return nil
+}
diff --git a/pkg/protocols/dns/request.go b/pkg/protocols/dns/request.go
index a16c2af88b..9457845270 100644
--- a/pkg/protocols/dns/request.go
+++ b/pkg/protocols/dns/request.go
@@ -106,7 +106,7 @@ func (request *Request) ExecuteWithResults(input *contextargs.Context, metadata,
}
func (request *Request) execute(input *contextargs.Context, domain string, metadata, previous output.InternalEvent, vars map[string]interface{}, callback protocols.OutputEventCallback) error {
-
+ var err error
if vardump.EnableVarDump {
gologger.Debug().Msgf("DNS Protocol request variables: \n%s\n", vardump.DumpVariables(vars))
}
@@ -199,7 +199,7 @@ func (request *Request) execute(input *contextargs.Context, domain string, metad
}
callback(event)
- return nil
+ return err
}
func (request *Request) parseDNSInput(host string) (string, error) {
diff --git a/pkg/protocols/network/request.go b/pkg/protocols/network/request.go
index 90390e53c3..5fa8609d51 100644
--- a/pkg/protocols/network/request.go
+++ b/pkg/protocols/network/request.go
@@ -155,14 +155,14 @@ func (request *Request) executeOnTarget(input *contextargs.Context, visited maps
}
visited.Set(actualAddress, struct{}{})
- if err := request.executeAddress(variables, actualAddress, address, input, kv.tls, previous, callback); err != nil {
+ if err = request.executeAddress(variables, actualAddress, address, input, kv.tls, previous, callback); err != nil {
outputEvent := request.responseToDSLMap("", "", "", address, "")
callback(&output.InternalWrappedEvent{InternalEvent: outputEvent})
gologger.Warning().Msgf("[%v] Could not make network request for (%s) : %s\n", request.options.TemplateID, actualAddress, err)
continue
}
}
- return nil
+ return err
}
// executeAddress executes the request for an address
diff --git a/pkg/protocols/network/request_test.go b/pkg/protocols/network/request_test.go
index 1945888e9b..7ff0f4882b 100644
--- a/pkg/protocols/network/request_test.go
+++ b/pkg/protocols/network/request_test.go
@@ -86,7 +86,7 @@ func TestNetworkExecuteWithResults(t *testing.T) {
err := request.ExecuteWithResults(ctxArgs, metadata, previous, func(event *output.InternalWrappedEvent) {
finalEvent = event
})
- require.Nil(t, err, "could not execute network request")
+ require.NotNil(t, err, "could not execute network request")
})
require.Nil(t, finalEvent.Results, "could not get event output from request")
diff --git a/pkg/scan/scan_context.go b/pkg/scan/scan_context.go
index b8f59ac7d4..45456ddcac 100644
--- a/pkg/scan/scan_context.go
+++ b/pkg/scan/scan_context.go
@@ -52,6 +52,10 @@ func (s *ScanContext) Context() context.Context {
return s.ctx
}
+func (s *ScanContext) GenerateErrorMessage() string {
+ return joinErrors(s.errors)
+}
+
// GenerateResult returns final results slice from all events
func (s *ScanContext) GenerateResult() []*output.ResultEvent {
s.m.Lock()
@@ -96,7 +100,7 @@ func (s *ScanContext) LogError(err error) {
}
s.errors = append(s.errors, err)
- errorMessage := joinErrors(s.errors)
+ errorMessage := s.GenerateErrorMessage()
for _, result := range s.results {
result.Error = errorMessage
diff --git a/pkg/tmplexec/exec.go b/pkg/tmplexec/exec.go
index 3d09f5e7a0..4ca9badf70 100644
--- a/pkg/tmplexec/exec.go
+++ b/pkg/tmplexec/exec.go
@@ -10,6 +10,7 @@ import (
"github.com/dop251/goja"
"github.com/projectdiscovery/gologger"
"github.com/projectdiscovery/nuclei/v3/pkg/js/compiler"
+ "github.com/projectdiscovery/nuclei/v3/pkg/operators"
"github.com/projectdiscovery/nuclei/v3/pkg/operators/common/dsl"
"github.com/projectdiscovery/nuclei/v3/pkg/output"
"github.com/projectdiscovery/nuclei/v3/pkg/protocols"
@@ -19,6 +20,8 @@ import (
"github.com/projectdiscovery/nuclei/v3/pkg/tmplexec/flow"
"github.com/projectdiscovery/nuclei/v3/pkg/tmplexec/generic"
"github.com/projectdiscovery/nuclei/v3/pkg/tmplexec/multiproto"
+ "github.com/projectdiscovery/nuclei/v3/pkg/types/nucleierr"
+ "github.com/projectdiscovery/utils/errkit"
)
// TemplateExecutor is an executor for a template
@@ -126,6 +129,8 @@ func (e *TemplateExecuter) Execute(ctx *scan.ScanContext) (bool, error) {
executed := &atomic.Bool{}
// matched in this case means something was exported / written to output
matched := &atomic.Bool{}
+ // callbackCalled tracks if the callback was called or not
+ callbackCalled := &atomic.Bool{}
defer func() {
// it is essential to remove template context of `Scan i.e template x input pair`
// since it is of no use after scan is completed (regardless of success or failure)
@@ -143,6 +148,7 @@ func (e *TemplateExecuter) Execute(ctx *scan.ScanContext) (bool, error) {
}
ctx.OnResult = func(event *output.InternalWrappedEvent) {
+ callbackCalled.Store(true)
if event == nil {
// something went wrong
return
@@ -198,13 +204,64 @@ func (e *TemplateExecuter) Execute(ctx *scan.ScanContext) (bool, error) {
} else {
errx = e.engine.ExecuteWithResults(ctx)
}
+ ctx.LogError(errx)
if lastMatcherEvent != nil {
+ lastMatcherEvent.InternalEvent["error"] = tryParseCause(fmt.Errorf("%s", ctx.GenerateErrorMessage()))
writeFailureCallback(lastMatcherEvent, e.options.Options.MatcherStatus)
}
+
+ //TODO: this is a hacky way to handle the case where the callback is not called and matcher-status is true.
+ // This is a workaround and needs to be refactored.
+ // Check if callback was never called and matcher-status is true
+ if !callbackCalled.Load() && e.options.Options.MatcherStatus {
+ fakeEvent := &output.InternalWrappedEvent{
+ Results: []*output.ResultEvent{
+ {
+ TemplateID: e.options.TemplateID,
+ Info: e.options.TemplateInfo,
+ Type: e.getTemplateType(),
+ Host: ctx.Input.MetaInput.Input,
+ Error: tryParseCause(fmt.Errorf("%s", ctx.GenerateErrorMessage())),
+ },
+ },
+ OperatorsResult: &operators.Result{
+ Matched: false,
+ },
+ }
+ writeFailureCallback(fakeEvent, e.options.Options.MatcherStatus)
+ }
+
return executed.Load() || matched.Load(), errx
}
+// tryParseCause tries to parse the cause of given error
+// this is legacy support due to use of errorutil in existing libraries
+// but this should not be required once all libraries are updated
+func tryParseCause(err error) string {
+ errStr := ""
+ errX := errkit.FromError(err)
+ if errX != nil {
+ var errCause error
+
+ if len(errX.Errors()) > 1 {
+ errCause = errX.Errors()[0]
+ }
+ if errCause == nil {
+ errCause = errX
+ }
+
+ msg := strings.Trim(errCause.Error(), "{} ")
+ parts := strings.Split(msg, ":")
+ errCause = errkit.New("%s", parts[len(parts)-1])
+ errKind := errkit.GetErrorKind(err, nucleierr.ErrTemplateLogic).String()
+ errStr = errCause.Error()
+ errStr = strings.TrimSpace(strings.Replace(errStr, "errKind="+errKind, "", -1))
+ }
+
+ return errStr
+}
+
// ExecuteWithResults executes the protocol requests and returns results instead of writing them.
func (e *TemplateExecuter) ExecuteWithResults(ctx *scan.ScanContext) ([]*output.ResultEvent, error) {
var errx error
From e0b254286806d4517ea8ce69b6cb25a044b5a1ef Mon Sep 17 00:00:00 2001
From: Dwi Siswanto
Date: Wed, 28 Aug 2024 19:27:45 +0700
Subject: [PATCH 177/283] feat: conditionally panic-recover (#5553)
* feat: conditionally panic-recover
As discussed with @Mzack9999, we should avoid
overusing panic-recover. We need to review the RCA
first to determine whether this is an exceptional
situation or if it's a higher-level function meant
to recover from a panic. This approach will help
us establish a robust error-handling strategy.
The implementation of panic-recover should be
conditional and NOT applied when running in a CI
environment AND IS temporary. Once we've caught
all errors and made the necessary corrections, we
can remove the deferred recover function.
Signed-off-by: Dwi Siswanto
* chore(deps): bump `go-ci` to v1.0.2
Signed-off-by: Dwi Siswanto
* chore(make): add `-race` to `GOFLAGS` in `test`
Signed-off-by: Dwi Siswanto
---------
Signed-off-by: Dwi Siswanto
---
Makefile | 1 +
go.mod | 1 +
go.sum | 4 ++++
pkg/js/compiler/compiler.go | 7 +++++++
pkg/js/compiler/pool.go | 8 ++++++++
pkg/protocols/headless/engine/page_actions.go | 6 ++++++
pkg/tmplexec/flow/flow_executor.go | 7 +++++++
7 files changed, 34 insertions(+)
diff --git a/Makefile b/Makefile
index 3fa6a033ef..8614ef82c9 100644
--- a/Makefile
+++ b/Makefile
@@ -65,6 +65,7 @@ docs:
git reset --hard # line 59
+test: GOFLAGS = -race -v
test:
$(GOTEST) $(GOFLAGS) ./...
diff --git a/go.mod b/go.mod
index a0cd13984a..858a21bca3 100644
--- a/go.mod
+++ b/go.mod
@@ -177,6 +177,7 @@ require (
github.com/jmespath/go-jmespath v0.4.0 // indirect
github.com/josharian/intern v1.0.0 // indirect
github.com/kataras/jwt v0.1.10 // indirect
+ github.com/kitabisa/go-ci v1.0.2 // indirect
github.com/klauspost/compress v1.17.8 // indirect
github.com/klauspost/pgzip v1.2.6 // indirect
github.com/kylelemons/godebug v1.1.0 // indirect
diff --git a/go.sum b/go.sum
index 310bd42ef8..66c91ae901 100644
--- a/go.sum
+++ b/go.sum
@@ -625,6 +625,10 @@ github.com/kevinburke/ssh_config v1.2.0/go.mod h1:CT57kijsi8u/K/BOFA39wgDQJ9CxiF
github.com/kisielk/errcheck v1.1.0/go.mod h1:EZBBE59ingxPouuu3KfxchcWSUPOHkagtvWXihfKN4Q=
github.com/kisielk/errcheck v1.5.0/go.mod h1:pFxgyoBC7bSaBwPgfKdkLd5X25qrDl4LWUI2bnpBCr8=
github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck=
+github.com/kitabisa/go-ci v1.0.1 h1://FHQzlDqYW+3qr0judsOE9X2ZrdRlRc66sCaVrLFGc=
+github.com/kitabisa/go-ci v1.0.1/go.mod h1:4MWu+kf/+tvd0vLWSJA689Kn+hrYkZiymmZYT5BGT4g=
+github.com/kitabisa/go-ci v1.0.2 h1:rqHf8KEbQOxVb998TbqGRo70Z7ol44io7/jLYJUvKp8=
+github.com/kitabisa/go-ci v1.0.2/go.mod h1:e3wBSzaJbcifXrr/Gw2ZBLn44MmeqP5WySwXyHlCK/U=
github.com/klauspost/compress v1.4.1/go.mod h1:RyIbtBH6LamlWaDj8nUwkbUhJ87Yi3uG0guNDohfE1A=
github.com/klauspost/compress v1.11.4/go.mod h1:aoV0uJVorq1K+umq18yTdKaF57EivdYsUV+/s2qKfXs=
github.com/klauspost/compress v1.17.8 h1:YcnTYrq7MikUT7k0Yb5eceMmALQPYBW/Xltxn0NAMnU=
diff --git a/pkg/js/compiler/compiler.go b/pkg/js/compiler/compiler.go
index c2dc15d2cb..b13e7f9ecb 100644
--- a/pkg/js/compiler/compiler.go
+++ b/pkg/js/compiler/compiler.go
@@ -6,6 +6,7 @@ import (
"fmt"
"github.com/dop251/goja"
+ "github.com/kitabisa/go-ci"
"github.com/projectdiscovery/nuclei/v3/pkg/protocols/common/generators"
"github.com/projectdiscovery/nuclei/v3/pkg/types"
@@ -120,11 +121,17 @@ func (c *Compiler) ExecuteWithOptions(program *goja.Program, args *ExecuteArgs,
defer cancel()
// execute the script
results, err := contextutil.ExecFuncWithTwoReturns(ctx, func() (val goja.Value, err error) {
+ // TODO(dwisiswant0): remove this once we get the RCA.
defer func() {
+ if ci.IsCI() {
+ return
+ }
+
if r := recover(); r != nil {
err = fmt.Errorf("panic: %v", r)
}
}()
+
return ExecuteProgram(program, args, opts)
})
if err != nil {
diff --git a/pkg/js/compiler/pool.go b/pkg/js/compiler/pool.go
index 3407d97369..31e2e5378f 100644
--- a/pkg/js/compiler/pool.go
+++ b/pkg/js/compiler/pool.go
@@ -11,6 +11,7 @@ import (
"github.com/dop251/goja"
"github.com/dop251/goja_nodejs/console"
"github.com/dop251/goja_nodejs/require"
+ "github.com/kitabisa/go-ci"
"github.com/projectdiscovery/gologger"
_ "github.com/projectdiscovery/nuclei/v3/pkg/js/generated/go/libbytes"
_ "github.com/projectdiscovery/nuclei/v3/pkg/js/generated/go/libfs"
@@ -84,11 +85,18 @@ func executeWithRuntime(runtime *goja.Runtime, p *goja.Program, args *ExecuteArg
opts.Cleanup(runtime)
}
}()
+
+ // TODO(dwisiswant0): remove this once we get the RCA.
defer func() {
+ if ci.IsCI() {
+ return
+ }
+
if r := recover(); r != nil {
err = fmt.Errorf("panic: %s", r)
}
}()
+
// set template ctx
_ = runtime.Set("template", args.TemplateCtx)
// set args
diff --git a/pkg/protocols/headless/engine/page_actions.go b/pkg/protocols/headless/engine/page_actions.go
index 57cc371093..576b6134fa 100644
--- a/pkg/protocols/headless/engine/page_actions.go
+++ b/pkg/protocols/headless/engine/page_actions.go
@@ -14,6 +14,7 @@ import (
"github.com/go-rod/rod/lib/input"
"github.com/go-rod/rod/lib/proto"
"github.com/go-rod/rod/lib/utils"
+ "github.com/kitabisa/go-ci"
"github.com/pkg/errors"
"github.com/projectdiscovery/gologger"
"github.com/projectdiscovery/nuclei/v3/pkg/protocols/common/contextargs"
@@ -54,7 +55,12 @@ func (p *Page) ExecuteActions(input *contextargs.Context, actions []*Action, var
waitFuncs := make([]func() error, 0)
// avoid any future panics caused due to go-rod library
+ // TODO(dwisiswant0): remove this once we get the RCA.
defer func() {
+ if ci.IsCI() {
+ return
+ }
+
if r := recover(); r != nil {
err = errorutil.New("panic on headless action: %v", r)
}
diff --git a/pkg/tmplexec/flow/flow_executor.go b/pkg/tmplexec/flow/flow_executor.go
index 6a1813efd9..6e71cf8407 100644
--- a/pkg/tmplexec/flow/flow_executor.go
+++ b/pkg/tmplexec/flow/flow_executor.go
@@ -13,6 +13,7 @@ import (
"github.com/projectdiscovery/nuclei/v3/pkg/scan"
templateTypes "github.com/projectdiscovery/nuclei/v3/pkg/templates/types"
+ "github.com/kitabisa/go-ci"
"github.com/projectdiscovery/nuclei/v3/pkg/types"
errorutil "github.com/projectdiscovery/utils/errors"
fileutil "github.com/projectdiscovery/utils/file"
@@ -201,7 +202,13 @@ func (f *FlowExecutor) ExecuteWithResults(ctx *scan.ScanContext) error {
}
}()
+
+ // TODO(dwisiswant0): remove this once we get the RCA.
defer func() {
+ if ci.IsCI() {
+ return
+ }
+
if r := recover(); r != nil {
f.ctx.LogError(fmt.Errorf("panic occurred while executing flow: %v", r))
}
From 88b281ae476bb7f93e84c501d86bb4fa31e0454a Mon Sep 17 00:00:00 2001
From: linchizhen
Date: Sun, 1 Sep 2024 16:41:01 +0800
Subject: [PATCH 178/283] chore: fix some function names in comment (#5586)
Signed-off-by: linchizhen
---
cmd/tmc/main.go | 2 +-
pkg/input/formats/openapi/generator.go | 2 +-
pkg/input/provider/interface.go | 2 +-
3 files changed, 3 insertions(+), 3 deletions(-)
diff --git a/cmd/tmc/main.go b/cmd/tmc/main.go
index 5cb38afbee..a5971ca193 100644
--- a/cmd/tmc/main.go
+++ b/cmd/tmc/main.go
@@ -300,7 +300,7 @@ func formatTemplate(data string) (string, bool, error) {
return data, false, errorutil.New("template format failed")
}
-// lintTemplateData lints template data using templateman lint api
+// lintTemplate lints template data using templateman lint api
func lintTemplate(data string) (bool, error) {
resp, err := retryablehttp.DefaultClient().Post(fmt.Sprintf("%s/lint", tmBaseUrl), "application/x-yaml", strings.NewReader(data))
if err != nil {
diff --git a/pkg/input/formats/openapi/generator.go b/pkg/input/formats/openapi/generator.go
index 4027d76fc7..63fef7e290 100644
--- a/pkg/input/formats/openapi/generator.go
+++ b/pkg/input/formats/openapi/generator.go
@@ -416,7 +416,7 @@ schemaLabel:
return globalParams, nil
}
-// generateExampleFromSchema generates an example from a schema object
+// GenerateParameterFromSecurityScheme generates an example from a schema object
func GenerateParameterFromSecurityScheme(scheme *openapi3.SecuritySchemeRef) (*openapi3.Parameter, error) {
if !generic.EqualsAny(scheme.Value.Type, "http", "apiKey") {
return nil, errorutil.NewWithTag("openapi", "unsupported security scheme type (%s) found in openapi file", scheme.Value.Type)
diff --git a/pkg/input/provider/interface.go b/pkg/input/provider/interface.go
index b7ad4b4480..e6d5da14a5 100644
--- a/pkg/input/provider/interface.go
+++ b/pkg/input/provider/interface.go
@@ -121,7 +121,7 @@ func NewInputProvider(opts InputOptions) (InputProvider, error) {
}
}
-// SupportedFormats returns all supported input formats of nuclei
+// SupportedInputFormats returns all supported input formats of nuclei
func SupportedInputFormats() string {
return "list, " + http.SupportedFormats()
}
From fdb8e0a8c68f0aaa6669848e035400ccae47a0fe Mon Sep 17 00:00:00 2001
From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com>
Date: Mon, 2 Sep 2024 05:11:44 +0000
Subject: [PATCH 179/283] chore(deps): bump
github.com/projectdiscovery/fastdialer
Bumps [github.com/projectdiscovery/fastdialer](https://github.com/projectdiscovery/fastdialer) from 0.2.5 to 0.2.7.
- [Release notes](https://github.com/projectdiscovery/fastdialer/releases)
- [Commits](https://github.com/projectdiscovery/fastdialer/compare/v0.2.5...v0.2.7)
---
updated-dependencies:
- dependency-name: github.com/projectdiscovery/fastdialer
dependency-type: direct:production
update-type: version-update:semver-patch
...
Signed-off-by: dependabot[bot]
---
go.mod | 8 ++++----
go.sum | 14 ++++++--------
2 files changed, 10 insertions(+), 12 deletions(-)
diff --git a/go.mod b/go.mod
index 858a21bca3..7b3b596b96 100644
--- a/go.mod
+++ b/go.mod
@@ -20,11 +20,11 @@ require (
github.com/olekukonko/tablewriter v0.0.5
github.com/pkg/errors v0.9.1
github.com/projectdiscovery/clistats v0.1.0
- github.com/projectdiscovery/fastdialer v0.2.5
- github.com/projectdiscovery/hmap v0.0.55
+ github.com/projectdiscovery/fastdialer v0.2.7
+ github.com/projectdiscovery/hmap v0.0.56
github.com/projectdiscovery/interactsh v1.2.0
github.com/projectdiscovery/rawhttp v0.1.60
- github.com/projectdiscovery/retryabledns v1.0.72
+ github.com/projectdiscovery/retryabledns v1.0.73
github.com/projectdiscovery/retryablehttp-go v1.0.74
github.com/projectdiscovery/yamldoc-go v1.0.4
github.com/remeh/sizedwaitgroup v1.0.0
@@ -72,6 +72,7 @@ require (
github.com/go-sql-driver/mysql v1.7.1
github.com/h2non/filetype v1.1.3
github.com/invopop/yaml v0.3.1
+ github.com/kitabisa/go-ci v1.0.2
github.com/labstack/echo/v4 v4.10.2
github.com/leslie-qiwa/flat v0.0.0-20230424180412-f9d1cf014baa
github.com/lib/pq v1.10.9
@@ -177,7 +178,6 @@ require (
github.com/jmespath/go-jmespath v0.4.0 // indirect
github.com/josharian/intern v1.0.0 // indirect
github.com/kataras/jwt v0.1.10 // indirect
- github.com/kitabisa/go-ci v1.0.2 // indirect
github.com/klauspost/compress v1.17.8 // indirect
github.com/klauspost/pgzip v1.2.6 // indirect
github.com/kylelemons/godebug v1.1.0 // indirect
diff --git a/go.sum b/go.sum
index 66c91ae901..2fedffa27a 100644
--- a/go.sum
+++ b/go.sum
@@ -625,8 +625,6 @@ github.com/kevinburke/ssh_config v1.2.0/go.mod h1:CT57kijsi8u/K/BOFA39wgDQJ9CxiF
github.com/kisielk/errcheck v1.1.0/go.mod h1:EZBBE59ingxPouuu3KfxchcWSUPOHkagtvWXihfKN4Q=
github.com/kisielk/errcheck v1.5.0/go.mod h1:pFxgyoBC7bSaBwPgfKdkLd5X25qrDl4LWUI2bnpBCr8=
github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck=
-github.com/kitabisa/go-ci v1.0.1 h1://FHQzlDqYW+3qr0judsOE9X2ZrdRlRc66sCaVrLFGc=
-github.com/kitabisa/go-ci v1.0.1/go.mod h1:4MWu+kf/+tvd0vLWSJA689Kn+hrYkZiymmZYT5BGT4g=
github.com/kitabisa/go-ci v1.0.2 h1:rqHf8KEbQOxVb998TbqGRo70Z7ol44io7/jLYJUvKp8=
github.com/kitabisa/go-ci v1.0.2/go.mod h1:e3wBSzaJbcifXrr/Gw2ZBLn44MmeqP5WySwXyHlCK/U=
github.com/klauspost/compress v1.4.1/go.mod h1:RyIbtBH6LamlWaDj8nUwkbUhJ87Yi3uG0guNDohfE1A=
@@ -836,8 +834,8 @@ github.com/projectdiscovery/clistats v0.1.0 h1:b+LF1w0xhNd7cneKWMXb+/yUmF1n5szaw
github.com/projectdiscovery/clistats v0.1.0/go.mod h1:GJ2av0KnOvK0AISQnP8hyDclYIji1LVkx2l0pwnzAu4=
github.com/projectdiscovery/dsl v0.1.8 h1:ulz+o097XsYgqP4QXaAQhVSkoeD2ZPWd29wX3CTodUA=
github.com/projectdiscovery/dsl v0.1.8/go.mod h1:AYJS2WQ/q0smr2v4pEJTg4DPe6k56KFKR7UFXvzNz/4=
-github.com/projectdiscovery/fastdialer v0.2.5 h1:OZg31LFIBBHbXVnOEcFNtD0gBM80zpY68QvYI0xbw80=
-github.com/projectdiscovery/fastdialer v0.2.5/go.mod h1:E1vmos7AeD/KZ7rsDeImjLp5H3jjas+2utVdUxGulvY=
+github.com/projectdiscovery/fastdialer v0.2.7 h1:x7bEf/kDyVOuG9kHfJLMhWO3h7J+6fMDgLWkGYL8EFg=
+github.com/projectdiscovery/fastdialer v0.2.7/go.mod h1:7/BGdaw7sCekuBh6lLZqPuoO5N0Qaizwvex/PjO+V8o=
github.com/projectdiscovery/fasttemplate v0.0.2 h1:h2cISk5xDhlJEinlBQS6RRx0vOlOirB2y3Yu4PJzpiA=
github.com/projectdiscovery/fasttemplate v0.0.2/go.mod h1:XYWWVMxnItd+r0GbjA1GCsUopMw1/XusuQxdyAIHMCw=
github.com/projectdiscovery/freeport v0.0.5 h1:jnd3Oqsl4S8n0KuFkE5Hm8WGDP24ITBvmyw5pFTHS8Q=
@@ -852,8 +850,8 @@ github.com/projectdiscovery/gostruct v0.0.2 h1:s8gP8ApugGM4go1pA+sVlPDXaWqNP5BBD
github.com/projectdiscovery/gostruct v0.0.2/go.mod h1:H86peL4HKwMXcQQtEa6lmC8FuD9XFt6gkNR0B/Mu5PE=
github.com/projectdiscovery/gozero v0.0.2 h1:8fJeaCjxL9tpm33uG/RsCQs6HGM/NE6eA3cjkilRQ+E=
github.com/projectdiscovery/gozero v0.0.2/go.mod h1:d8bZvDWW07LWNYWrwjZ4OO1I0cpkfqaysyDfSs9ibK8=
-github.com/projectdiscovery/hmap v0.0.55 h1:By0qSeQRE/ybeITUEL6oURN+j7qX7FVjCq92duwsqcA=
-github.com/projectdiscovery/hmap v0.0.55/go.mod h1:uZOF/W8yOGDb/t9omqATirhVwiboa5F/c00OVhDeE0E=
+github.com/projectdiscovery/hmap v0.0.56 h1:QLIuFhCc7jrBH9WYpxNBDooSoH+/S8cCm7VO5rMt3lU=
+github.com/projectdiscovery/hmap v0.0.56/go.mod h1:XyGKMO93nLOzpaTTCDBSKEGx/XWXjSpoyZJxT92Qd/Q=
github.com/projectdiscovery/httpx v1.6.7 h1:luJAtQ+iKpiyKKDpcWw1/fF1MaRX6JZ4R3P+ol5Uikk=
github.com/projectdiscovery/httpx v1.6.7/go.mod h1:rqcuexBrb4v36ZyX0BHcHaovTurOJn2P65Tdt9cYdfE=
github.com/projectdiscovery/interactsh v1.2.0 h1:Al6jHiR+Usl9egYJDLJaWNHOcH8Rugk8gWMasc8Cmw8=
@@ -874,8 +872,8 @@ github.com/projectdiscovery/rawhttp v0.1.60 h1:L57kAATGRuCfOp8Fs0ly3j2wlip7R5pco
github.com/projectdiscovery/rawhttp v0.1.60/go.mod h1:M97w6lsphTHKeqLSio4Lw9K4KLO2riJMYAOqVMIZ3ck=
github.com/projectdiscovery/rdap v0.9.1-0.20221108103045-9865884d1917 h1:m03X4gBVSorSzvmm0bFa7gDV4QNSOWPL/fgZ4kTXBxk=
github.com/projectdiscovery/rdap v0.9.1-0.20221108103045-9865884d1917/go.mod h1:JxXtZC9e195awe7EynrcnBJmFoad/BNDzW9mzFkK8Sg=
-github.com/projectdiscovery/retryabledns v1.0.72 h1:+k9JRYBMpZIyEQZJWVHoCHx1TEdpiuAS7iEru2bhs3A=
-github.com/projectdiscovery/retryabledns v1.0.72/go.mod h1:BDhYnqSH0Os2G37eb4L4vnKrUcik6Q6pR02Cq5PbT+0=
+github.com/projectdiscovery/retryabledns v1.0.73 h1:CLqtgoI6+YVcUE2jnZPFVvwj59CHDE4HNImLf62sSuw=
+github.com/projectdiscovery/retryabledns v1.0.73/go.mod h1:Wo5tQHKtx6XcYYhMypRg8dr0GGnqIb4CzoTSEw2lo7I=
github.com/projectdiscovery/retryablehttp-go v1.0.74 h1:jXZsg3nNDuDjGNeL0+hiEx/dqUqPyZW1zsotWW1y70c=
github.com/projectdiscovery/retryablehttp-go v1.0.74/go.mod h1:fVRxa2fVMbomaPiOFA05kM2WrjV8wPFHW7J84eenVmQ=
github.com/projectdiscovery/sarif v0.0.1 h1:C2Tyj0SGOKbCLgHrx83vaE6YkzXEVrMXYRGLkKCr/us=
From 841d8913e574b75dac3feb7c5e418d9d959a84eb Mon Sep 17 00:00:00 2001
From: Dwi Siswanto
Date: Mon, 2 Sep 2024 16:59:52 +0700
Subject: [PATCH 180/283] feat(headless): add `ActionWaitDialog` type (#5545)
* feat(headless): add `dialog` action type
also implement it
Signed-off-by: Dwi Siswanto
* refactor(headless): add `ActionData` for action output datas
Signed-off-by: Dwi Siswanto
* refactor(headless): rm `value` arg for `*Page.HandleDialog`
also:
* expose `err` from \*proto.PageHandleJavaScriptDialog`
* conditional ActionData assignment based on
Signed-off-by: Dwi Siswanto
* refactor(headless): rename to `ActionWaitDialog`
Signed-off-by: Dwi Siswanto
* test(headless): fix mismatch assertion of `src` output of `ActionGetResource`
Signed-off-by: Dwi Siswanto
* test(headless): add TestActionWaitDialog test case
Signed-off-by: Dwi Siswanto
* feat(headless): add `GetActionDataWithDefault` generic func
Signed-off-by: Dwi Siswanto
* feat(headless): implement `GetActionDataWithDefault`
to `header` & `status_code`
Signed-off-by: Dwi Siswanto
* refactor(headless): use `mapsutil.Map` instead
Signed-off-by: Dwi Siswanto
* Revert "feat(headless): add `GetActionDataWithDefault` generic func"
This reverts commit fa12e0d6a221c8a7bf62200f69814ee27681f08f.
---------
Signed-off-by: Dwi Siswanto
---
pkg/protocols/headless/engine/action_types.go | 9 ++
pkg/protocols/headless/engine/page.go | 2 +-
pkg/protocols/headless/engine/page_actions.go | 87 +++++++++----
.../headless/engine/page_actions_test.go | 118 ++++++++++++++----
pkg/protocols/headless/request.go | 8 +-
5 files changed, 171 insertions(+), 53 deletions(-)
diff --git a/pkg/protocols/headless/engine/action_types.go b/pkg/protocols/headless/engine/action_types.go
index c9cd39ad2a..d61e377431 100644
--- a/pkg/protocols/headless/engine/action_types.go
+++ b/pkg/protocols/headless/engine/action_types.go
@@ -6,11 +6,15 @@ import (
"strings"
"github.com/invopop/jsonschema"
+ mapsutil "github.com/projectdiscovery/utils/maps"
)
// ActionType defines the action type for a browser action
type ActionType int8
+// ActionData stores the action output data
+type ActionData = mapsutil.Map[string, any]
+
// Types to be executed by the user.
// name:ActionType
const (
@@ -68,6 +72,9 @@ const (
// ActionWaitEvent waits for a specific event.
// name:waitevent
ActionWaitEvent
+ // ActionWaitDialog waits for JavaScript dialog (alert, confirm, prompt, or onbeforeunload).
+ // name:dialog
+ ActionWaitDialog
// ActionKeyboard performs a keyboard action event on a page.
// name:keyboard
ActionKeyboard
@@ -104,6 +111,7 @@ var ActionStringToAction = map[string]ActionType{
"deleteheader": ActionDeleteHeader,
"setbody": ActionSetBody,
"waitevent": ActionWaitEvent,
+ "waitdialog": ActionWaitDialog,
"keyboard": ActionKeyboard,
"debug": ActionDebug,
"sleep": ActionSleep,
@@ -130,6 +138,7 @@ var ActionToActionString = map[ActionType]string{
ActionDeleteHeader: "deleteheader",
ActionSetBody: "setbody",
ActionWaitEvent: "waitevent",
+ ActionWaitDialog: "waitdialog",
ActionKeyboard: "keyboard",
ActionDebug: "debug",
ActionSleep: "sleep",
diff --git a/pkg/protocols/headless/engine/page.go b/pkg/protocols/headless/engine/page.go
index 7e2bb37bbe..1b417ec402 100644
--- a/pkg/protocols/headless/engine/page.go
+++ b/pkg/protocols/headless/engine/page.go
@@ -45,7 +45,7 @@ type Options struct {
}
// Run runs a list of actions by creating a new page in the browser.
-func (i *Instance) Run(input *contextargs.Context, actions []*Action, payloads map[string]interface{}, options *Options) (map[string]string, *Page, error) {
+func (i *Instance) Run(input *contextargs.Context, actions []*Action, payloads map[string]interface{}, options *Options) (ActionData, *Page, error) {
page, err := i.engine.Page(proto.TargetCreateTarget{})
if err != nil {
return nil, nil, err
diff --git a/pkg/protocols/headless/engine/page_actions.go b/pkg/protocols/headless/engine/page_actions.go
index 576b6134fa..b34ed18b94 100644
--- a/pkg/protocols/headless/engine/page_actions.go
+++ b/pkg/protocols/headless/engine/page_actions.go
@@ -48,8 +48,8 @@ const (
)
// ExecuteActions executes a list of actions on a page.
-func (p *Page) ExecuteActions(input *contextargs.Context, actions []*Action, variables map[string]interface{}) (outData map[string]string, err error) {
- outData = make(map[string]string)
+func (p *Page) ExecuteActions(input *contextargs.Context, actions []*Action, variables map[string]interface{}) (outData ActionData, err error) {
+ outData = make(ActionData)
// waitFuncs are function that needs to be executed after navigation
// typically used for waitEvent
waitFuncs := make([]func() error, 0)
@@ -106,6 +106,8 @@ func (p *Page) ExecuteActions(input *contextargs.Context, actions []*Action, var
if waitFunc != nil {
waitFuncs = append(waitFuncs, waitFunc)
}
+ case ActionWaitDialog:
+ err = p.HandleDialog(act, outData)
case ActionFilesInput:
if p.options.Options.AllowLocalFileAccess {
err = p.FilesInput(act, outData)
@@ -148,7 +150,7 @@ type rule struct {
}
// WaitVisible waits until an element appears.
-func (p *Page) WaitVisible(act *Action, out map[string]string) error {
+func (p *Page) WaitVisible(act *Action, out ActionData) error {
timeout, err := getTimeout(p, act)
if err != nil {
return errors.Wrap(err, "Wrong timeout given")
@@ -223,7 +225,7 @@ func geTimeParameter(p *Page, act *Action, parameterName string, defaultValue ti
}
// ActionAddHeader executes a AddHeader action.
-func (p *Page) ActionAddHeader(act *Action, out map[string]string) error {
+func (p *Page) ActionAddHeader(act *Action, out ActionData) error {
in := p.getActionArgWithDefaultValues(act, "part")
args := make(map[string]string)
@@ -234,7 +236,7 @@ func (p *Page) ActionAddHeader(act *Action, out map[string]string) error {
}
// ActionSetHeader executes a SetHeader action.
-func (p *Page) ActionSetHeader(act *Action, out map[string]string) error {
+func (p *Page) ActionSetHeader(act *Action, out ActionData) error {
in := p.getActionArgWithDefaultValues(act, "part")
args := make(map[string]string)
@@ -245,7 +247,7 @@ func (p *Page) ActionSetHeader(act *Action, out map[string]string) error {
}
// ActionDeleteHeader executes a DeleteHeader action.
-func (p *Page) ActionDeleteHeader(act *Action, out map[string]string) error {
+func (p *Page) ActionDeleteHeader(act *Action, out ActionData) error {
in := p.getActionArgWithDefaultValues(act, "part")
args := make(map[string]string)
@@ -255,7 +257,7 @@ func (p *Page) ActionDeleteHeader(act *Action, out map[string]string) error {
}
// ActionSetBody executes a SetBody action.
-func (p *Page) ActionSetBody(act *Action, out map[string]string) error {
+func (p *Page) ActionSetBody(act *Action, out ActionData) error {
in := p.getActionArgWithDefaultValues(act, "part")
args := make(map[string]string)
@@ -265,7 +267,7 @@ func (p *Page) ActionSetBody(act *Action, out map[string]string) error {
}
// ActionSetMethod executes an SetMethod action.
-func (p *Page) ActionSetMethod(act *Action, out map[string]string) error {
+func (p *Page) ActionSetMethod(act *Action, out ActionData) error {
in := p.getActionArgWithDefaultValues(act, "part")
args := make(map[string]string)
@@ -275,7 +277,7 @@ func (p *Page) ActionSetMethod(act *Action, out map[string]string) error {
}
// NavigateURL executes an ActionLoadURL actions loading a URL for the page.
-func (p *Page) NavigateURL(action *Action, out map[string]string, allvars map[string]interface{}) error {
+func (p *Page) NavigateURL(action *Action, out ActionData, allvars map[string]interface{}) error {
// input <- is input url from cli
// target <- is the url from template (ex: {{BaseURL}}/test)
input, err := urlutil.Parse(p.input.MetaInput.Input)
@@ -331,7 +333,7 @@ func (p *Page) NavigateURL(action *Action, out map[string]string, allvars map[st
}
// RunScript runs a script on the loaded page
-func (p *Page) RunScript(action *Action, out map[string]string) error {
+func (p *Page) RunScript(action *Action, out ActionData) error {
code := p.getActionArgWithDefaultValues(action, "code")
if code == "" {
return errinvalidArguments
@@ -352,7 +354,7 @@ func (p *Page) RunScript(action *Action, out map[string]string) error {
}
// ClickElement executes click actions for an element.
-func (p *Page) ClickElement(act *Action, out map[string]string) error {
+func (p *Page) ClickElement(act *Action, out ActionData) error {
element, err := p.pageElementBy(act.Data)
if err != nil {
return errors.Wrap(err, errCouldNotGetElement)
@@ -367,12 +369,12 @@ func (p *Page) ClickElement(act *Action, out map[string]string) error {
}
// KeyboardAction executes a keyboard action on the page.
-func (p *Page) KeyboardAction(act *Action, out map[string]string) error {
+func (p *Page) KeyboardAction(act *Action, out ActionData) error {
return p.page.Keyboard.Type([]input.Key(p.getActionArgWithDefaultValues(act, "keys"))...)
}
// RightClickElement executes right click actions for an element.
-func (p *Page) RightClickElement(act *Action, out map[string]string) error {
+func (p *Page) RightClickElement(act *Action, out ActionData) error {
element, err := p.pageElementBy(act.Data)
if err != nil {
return errors.Wrap(err, errCouldNotGetElement)
@@ -387,7 +389,7 @@ func (p *Page) RightClickElement(act *Action, out map[string]string) error {
}
// Screenshot executes screenshot action on a page
-func (p *Page) Screenshot(act *Action, out map[string]string) error {
+func (p *Page) Screenshot(act *Action, out ActionData) error {
to := p.getActionArgWithDefaultValues(act, "to")
if to == "" {
to = ksuid.New().String()
@@ -450,7 +452,7 @@ func (p *Page) Screenshot(act *Action, out map[string]string) error {
}
// InputElement executes input element actions for an element.
-func (p *Page) InputElement(act *Action, out map[string]string) error {
+func (p *Page) InputElement(act *Action, out ActionData) error {
value := p.getActionArgWithDefaultValues(act, "value")
if value == "" {
return errinvalidArguments
@@ -469,7 +471,7 @@ func (p *Page) InputElement(act *Action, out map[string]string) error {
}
// TimeInputElement executes time input on an element
-func (p *Page) TimeInputElement(act *Action, out map[string]string) error {
+func (p *Page) TimeInputElement(act *Action, out ActionData) error {
value := p.getActionArgWithDefaultValues(act, "value")
if value == "" {
return errinvalidArguments
@@ -492,7 +494,7 @@ func (p *Page) TimeInputElement(act *Action, out map[string]string) error {
}
// SelectInputElement executes select input statement action on a element
-func (p *Page) SelectInputElement(act *Action, out map[string]string) error {
+func (p *Page) SelectInputElement(act *Action, out ActionData) error {
value := p.getActionArgWithDefaultValues(act, "value")
if value == "" {
return errinvalidArguments
@@ -517,7 +519,7 @@ func (p *Page) SelectInputElement(act *Action, out map[string]string) error {
}
// WaitLoad waits for the page to load
-func (p *Page) WaitLoad(act *Action, out map[string]string) error {
+func (p *Page) WaitLoad(act *Action, out ActionData) error {
p.page.Timeout(2 * time.Second).WaitNavigation(proto.PageLifecycleEventNameFirstMeaningfulPaint)()
// Wait for the window.onload event and also wait for the network requests
@@ -531,7 +533,7 @@ func (p *Page) WaitLoad(act *Action, out map[string]string) error {
}
// GetResource gets a resource from an element from page.
-func (p *Page) GetResource(act *Action, out map[string]string) error {
+func (p *Page) GetResource(act *Action, out ActionData) error {
element, err := p.pageElementBy(act.Data)
if err != nil {
return errors.Wrap(err, errCouldNotGetElement)
@@ -547,7 +549,7 @@ func (p *Page) GetResource(act *Action, out map[string]string) error {
}
// FilesInput acts with a file input element on page
-func (p *Page) FilesInput(act *Action, out map[string]string) error {
+func (p *Page) FilesInput(act *Action, out ActionData) error {
element, err := p.pageElementBy(act.Data)
if err != nil {
return errors.Wrap(err, errCouldNotGetElement)
@@ -564,7 +566,7 @@ func (p *Page) FilesInput(act *Action, out map[string]string) error {
}
// ExtractElement extracts from an element on the page.
-func (p *Page) ExtractElement(act *Action, out map[string]string) error {
+func (p *Page) ExtractElement(act *Action, out ActionData) error {
element, err := p.pageElementBy(act.Data)
if err != nil {
return errors.Wrap(err, errCouldNotGetElement)
@@ -598,7 +600,7 @@ func (p *Page) ExtractElement(act *Action, out map[string]string) error {
}
// WaitEvent waits for an event to happen on the page.
-func (p *Page) WaitEvent(act *Action, out map[string]string) (func() error, error) {
+func (p *Page) WaitEvent(act *Action, out ActionData) (func() error, error) {
event := p.getActionArgWithDefaultValues(act, "event")
if event == "" {
return nil, errors.New("event not recognized")
@@ -636,6 +638,43 @@ func (p *Page) WaitEvent(act *Action, out map[string]string) (func() error, erro
return waitFunc, nil
}
+// HandleDialog handles JavaScript dialog (alert, confirm, prompt, or onbeforeunload).
+func (p *Page) HandleDialog(act *Action, out ActionData) error {
+ maxDuration := 10 * time.Second
+
+ if dur := p.getActionArgWithDefaultValues(act, "max-duration"); dur != "" {
+ var err error
+
+ maxDuration, err = time.ParseDuration(dur)
+ if err != nil {
+ return errorutil.NewWithErr(err).Msgf("could not parse max-duration")
+ }
+ }
+
+ ctx, cancel := context.WithTimeout(context.Background(), maxDuration)
+ defer cancel()
+
+ wait, handle := p.page.HandleDialog()
+ fn := func() (*proto.PageJavascriptDialogOpening, error) {
+ dialog := wait()
+ err := handle(&proto.PageHandleJavaScriptDialog{
+ Accept: true,
+ PromptText: "",
+ })
+
+ return dialog, err
+ }
+
+ dialog, err := contextutil.ExecFuncWithTwoReturns(ctx, fn)
+ if err == nil && act.Name != "" {
+ out[act.Name] = true
+ out[act.Name+"_type"] = string(dialog.Type)
+ out[act.Name+"_message"] = dialog.Message
+ }
+
+ return nil
+}
+
// pageElementBy returns a page element from a variety of inputs.
//
// Supported values for by: r -> selector & regex, x -> xpath, js -> eval js,
@@ -670,14 +709,14 @@ func (p *Page) pageElementBy(data map[string]string) (*rod.Element, error) {
}
// DebugAction enables debug action on a page.
-func (p *Page) DebugAction(act *Action, out map[string]string) error {
+func (p *Page) DebugAction(act *Action, out ActionData) error {
p.instance.browser.engine.SlowMotion(5 * time.Second)
p.instance.browser.engine.Trace(true)
return nil
}
// SleepAction sleeps on the page for a specified duration
-func (p *Page) SleepAction(act *Action, out map[string]string) error {
+func (p *Page) SleepAction(act *Action, out ActionData) error {
seconds := act.Data["duration"]
if seconds == "" {
seconds = "5"
diff --git a/pkg/protocols/headless/engine/page_actions_test.go b/pkg/protocols/headless/engine/page_actions_test.go
index e6699d6408..3620bb2bb4 100644
--- a/pkg/protocols/headless/engine/page_actions_test.go
+++ b/pkg/protocols/headless/engine/page_actions_test.go
@@ -38,7 +38,7 @@ func TestActionNavigate(t *testing.T) {
actions := []*Action{{ActionType: ActionTypeHolder{ActionType: ActionNavigate}, Data: map[string]string{"url": "{{BaseURL}}"}}, {ActionType: ActionTypeHolder{ActionType: ActionWaitLoad}}}
- testHeadlessSimpleResponse(t, response, actions, 20*time.Second, func(page *Page, err error, out map[string]string) {
+ testHeadlessSimpleResponse(t, response, actions, 20*time.Second, func(page *Page, err error, out ActionData) {
require.Nilf(t, err, "could not run page actions")
require.Equal(t, "Nuclei Test Page", page.Page().MustInfo().Title, "could not navigate correctly")
})
@@ -63,7 +63,7 @@ func TestActionScript(t *testing.T) {
{ActionType: ActionTypeHolder{ActionType: ActionScript}, Name: "test", Data: map[string]string{"code": "() => window.test"}},
}
- testHeadlessSimpleResponse(t, response, actions, timeout, func(page *Page, err error, out map[string]string) {
+ testHeadlessSimpleResponse(t, response, actions, timeout, func(page *Page, err error, out ActionData) {
require.Nil(t, err, "could not run page actions")
require.Equal(t, "Nuclei Test Page", page.Page().MustInfo().Title, "could not navigate correctly")
require.Equal(t, "some-data", out["test"], "could not run js and get results correctly")
@@ -77,7 +77,7 @@ func TestActionScript(t *testing.T) {
{ActionType: ActionTypeHolder{ActionType: ActionWaitLoad}},
{ActionType: ActionTypeHolder{ActionType: ActionScript}, Name: "test", Data: map[string]string{"code": "() => window.test"}},
}
- testHeadlessSimpleResponse(t, response, actions, timeout, func(page *Page, err error, out map[string]string) {
+ testHeadlessSimpleResponse(t, response, actions, timeout, func(page *Page, err error, out ActionData) {
require.Nil(t, err, "could not run page actions")
require.Equal(t, "Nuclei Test Page", page.Page().MustInfo().Title, "could not navigate correctly")
require.Equal(t, "some-data", out["test"], "could not run js and get results correctly with js hook")
@@ -101,7 +101,7 @@ func TestActionClick(t *testing.T) {
{ActionType: ActionTypeHolder{ActionType: ActionClick}, Data: map[string]string{"selector": "button"}}, // Use css selector for clicking
}
- testHeadlessSimpleResponse(t, response, actions, 20*time.Second, func(page *Page, err error, out map[string]string) {
+ testHeadlessSimpleResponse(t, response, actions, 20*time.Second, func(page *Page, err error, out ActionData) {
require.Nil(t, err, "could not run page actions")
require.Equal(t, "Nuclei Test Page", page.Page().MustInfo().Title, "could not navigate correctly")
el := page.Page().MustElement("button")
@@ -134,7 +134,7 @@ func TestActionRightClick(t *testing.T) {
{ActionType: ActionTypeHolder{ActionType: ActionRightClick}, Data: map[string]string{"selector": "button"}}, // Use css selector for clicking
}
- testHeadlessSimpleResponse(t, response, actions, 20*time.Second, func(page *Page, err error, out map[string]string) {
+ testHeadlessSimpleResponse(t, response, actions, 20*time.Second, func(page *Page, err error, out ActionData) {
require.Nil(t, err, "could not run page actions")
require.Equal(t, "Nuclei Test Page", page.Page().MustInfo().Title, "could not navigate correctly")
el := page.Page().MustElement("button")
@@ -159,7 +159,7 @@ func TestActionTextInput(t *testing.T) {
{ActionType: ActionTypeHolder{ActionType: ActionTextInput}, Data: map[string]string{"selector": "input", "value": "test"}},
}
- testHeadlessSimpleResponse(t, response, actions, 20*time.Second, func(page *Page, err error, out map[string]string) {
+ testHeadlessSimpleResponse(t, response, actions, 20*time.Second, func(page *Page, err error, out ActionData) {
require.Nil(t, err, "could not run page actions")
require.Equal(t, "Nuclei Test Page", page.Page().MustInfo().Title, "could not navigate correctly")
el := page.Page().MustElement("input")
@@ -182,7 +182,7 @@ func TestActionHeadersChange(t *testing.T) {
}
}
- testHeadless(t, actions, 20*time.Second, handler, func(page *Page, err error, out map[string]string) {
+ testHeadless(t, actions, 20*time.Second, handler, func(page *Page, err error, out ActionData) {
require.Nil(t, err, "could not run page actions")
require.Equal(t, "found", strings.ToLower(strings.TrimSpace(page.Page().MustElement("html").MustText())), "could not set header correctly")
})
@@ -205,7 +205,7 @@ func TestActionScreenshot(t *testing.T) {
{ActionType: ActionTypeHolder{ActionType: ActionScreenshot}, Data: map[string]string{"to": filePath}},
}
- testHeadlessSimpleResponse(t, response, actions, 20*time.Second, func(page *Page, err error, out map[string]string) {
+ testHeadlessSimpleResponse(t, response, actions, 20*time.Second, func(page *Page, err error, out ActionData) {
require.Nil(t, err, "could not run page actions")
require.Equal(t, "Nuclei Test Page", page.Page().MustInfo().Title, "could not navigate correctly")
_ = page.Page()
@@ -233,7 +233,7 @@ func TestActionScreenshotToDir(t *testing.T) {
{ActionType: ActionTypeHolder{ActionType: ActionScreenshot}, Data: map[string]string{"to": filePath, "mkdir": "true"}},
}
- testHeadlessSimpleResponse(t, response, actions, 20*time.Second, func(page *Page, err error, out map[string]string) {
+ testHeadlessSimpleResponse(t, response, actions, 20*time.Second, func(page *Page, err error, out ActionData) {
require.Nil(t, err, "could not run page actions")
require.Equal(t, "Nuclei Test Page", page.Page().MustInfo().Title, "could not navigate correctly")
_ = page.Page()
@@ -260,7 +260,7 @@ func TestActionTimeInput(t *testing.T) {
{ActionType: ActionTypeHolder{ActionType: ActionTimeInput}, Data: map[string]string{"selector": "input", "value": "2006-01-02T15:04:05Z"}},
}
- testHeadlessSimpleResponse(t, response, actions, 20*time.Second, func(page *Page, err error, out map[string]string) {
+ testHeadlessSimpleResponse(t, response, actions, 20*time.Second, func(page *Page, err error, out ActionData) {
require.Nil(t, err, "could not run page actions")
require.Equal(t, "Nuclei Test Page", page.Page().MustInfo().Title, "could not navigate correctly")
el := page.Page().MustElement("input")
@@ -288,7 +288,7 @@ func TestActionSelectInput(t *testing.T) {
{ActionType: ActionTypeHolder{ActionType: ActionSelectInput}, Data: map[string]string{"by": "x", "xpath": "//select[@id='test']", "value": "Test2", "selected": "true"}},
}
- testHeadlessSimpleResponse(t, response, actions, 20*time.Second, func(page *Page, err error, out map[string]string) {
+ testHeadlessSimpleResponse(t, response, actions, 20*time.Second, func(page *Page, err error, out ActionData) {
require.Nil(t, err, "could not run page actions")
el := page.Page().MustElement("select")
require.Equal(t, "Test2", el.MustText(), "could not get input change value")
@@ -311,7 +311,7 @@ func TestActionFilesInput(t *testing.T) {
{ActionType: ActionTypeHolder{ActionType: ActionFilesInput}, Data: map[string]string{"selector": "input", "value": "test1.pdf"}},
}
- testHeadlessSimpleResponse(t, response, actions, 20*time.Second, func(page *Page, err error, out map[string]string) {
+ testHeadlessSimpleResponse(t, response, actions, 20*time.Second, func(page *Page, err error, out ActionData) {
require.Nil(t, err, "could not run page actions")
require.Equal(t, "Nuclei Test Page", page.Page().MustInfo().Title, "could not navigate correctly")
el := page.Page().MustElement("input")
@@ -337,7 +337,7 @@ func TestActionFilesInputNegative(t *testing.T) {
}
t.Setenv("LOCAL_FILE_ACCESS", "false")
- testHeadlessSimpleResponse(t, response, actions, 20*time.Second, func(page *Page, err error, out map[string]string) {
+ testHeadlessSimpleResponse(t, response, actions, 20*time.Second, func(page *Page, err error, out ActionData) {
require.ErrorContains(t, err, ErrLFAccessDenied.Error(), "got file access when -lfa is false")
})
}
@@ -359,7 +359,7 @@ func TestActionWaitLoad(t *testing.T) {
{ActionType: ActionTypeHolder{ActionType: ActionWaitLoad}},
}
- testHeadlessSimpleResponse(t, response, actions, 20*time.Second, func(page *Page, err error, out map[string]string) {
+ testHeadlessSimpleResponse(t, response, actions, 20*time.Second, func(page *Page, err error, out ActionData) {
require.Nil(t, err, "could not run page actions")
el := page.Page().MustElement("button")
style, attributeErr := el.Attribute("style")
@@ -384,9 +384,12 @@ func TestActionGetResource(t *testing.T) {
{ActionType: ActionTypeHolder{ActionType: ActionGetResource}, Data: map[string]string{"by": "x", "xpath": "//img[@id='test']"}, Name: "src"},
}
- testHeadlessSimpleResponse(t, response, actions, 20*time.Second, func(page *Page, err error, out map[string]string) {
+ testHeadlessSimpleResponse(t, response, actions, 20*time.Second, func(page *Page, err error, out ActionData) {
require.Nil(t, err, "could not run page actions")
- require.Equal(t, len(out["src"]), 121808, "could not find resource")
+
+ src, ok := out["src"].(string)
+ require.True(t, ok, "could not assert src to string")
+ require.Equal(t, len(src), 121808, "could not find resource")
})
}
@@ -404,7 +407,7 @@ func TestActionExtract(t *testing.T) {
{ActionType: ActionTypeHolder{ActionType: ActionExtract}, Data: map[string]string{"by": "x", "xpath": "//button[@id='test']"}, Name: "extract"},
}
- testHeadlessSimpleResponse(t, response, actions, 20*time.Second, func(page *Page, err error, out map[string]string) {
+ testHeadlessSimpleResponse(t, response, actions, 20*time.Second, func(page *Page, err error, out ActionData) {
require.Nil(t, err, "could not run page actions")
require.Equal(t, "Wait for me!", out["extract"], "could not extract text")
})
@@ -423,7 +426,7 @@ func TestActionSetMethod(t *testing.T) {
{ActionType: ActionTypeHolder{ActionType: ActionSetMethod}, Data: map[string]string{"part": "x", "method": "SET"}},
}
- testHeadlessSimpleResponse(t, response, actions, 20*time.Second, func(page *Page, err error, out map[string]string) {
+ testHeadlessSimpleResponse(t, response, actions, 20*time.Second, func(page *Page, err error, out ActionData) {
require.Nil(t, err, "could not run page actions")
require.Equal(t, "SET", page.rules[0].Args["method"], "could not find resource")
})
@@ -442,7 +445,7 @@ func TestActionAddHeader(t *testing.T) {
}
}
- testHeadless(t, actions, 20*time.Second, handler, func(page *Page, err error, out map[string]string) {
+ testHeadless(t, actions, 20*time.Second, handler, func(page *Page, err error, out ActionData) {
require.Nil(t, err, "could not run page actions")
require.Equal(t, "found", strings.ToLower(strings.TrimSpace(page.Page().MustElement("html").MustText())), "could not set header correctly")
})
@@ -463,7 +466,7 @@ func TestActionDeleteHeader(t *testing.T) {
}
}
- testHeadless(t, actions, 20*time.Second, handler, func(page *Page, err error, out map[string]string) {
+ testHeadless(t, actions, 20*time.Second, handler, func(page *Page, err error, out ActionData) {
require.Nil(t, err, "could not run page actions")
require.Equal(t, "header deleted", strings.ToLower(strings.TrimSpace(page.Page().MustElement("html").MustText())), "could not delete header correctly")
})
@@ -481,7 +484,7 @@ func TestActionSetBody(t *testing.T) {
_, _ = fmt.Fprintln(w, string(body))
}
- testHeadless(t, actions, 20*time.Second, handler, func(page *Page, err error, out map[string]string) {
+ testHeadless(t, actions, 20*time.Second, handler, func(page *Page, err error, out ActionData) {
require.Nil(t, err, "could not run page actions")
require.Equal(t, "hello", strings.ToLower(strings.TrimSpace(page.Page().MustElement("html").MustText())), "could not set header correctly")
})
@@ -505,7 +508,7 @@ func TestActionKeyboard(t *testing.T) {
{ActionType: ActionTypeHolder{ActionType: ActionKeyboard}, Data: map[string]string{"keys": "Test2"}},
}
- testHeadlessSimpleResponse(t, response, actions, 20*time.Second, func(page *Page, err error, out map[string]string) {
+ testHeadlessSimpleResponse(t, response, actions, 20*time.Second, func(page *Page, err error, out ActionData) {
require.Nil(t, err, "could not run page actions")
el := page.Page().MustElement("input")
require.Equal(t, "Test2", el.MustText(), "could not get input change value")
@@ -529,7 +532,7 @@ func TestActionSleep(t *testing.T) {
{ActionType: ActionTypeHolder{ActionType: ActionSleep}, Data: map[string]string{"duration": "2"}},
}
- testHeadlessSimpleResponse(t, response, actions, 20*time.Second, func(page *Page, err error, out map[string]string) {
+ testHeadlessSimpleResponse(t, response, actions, 20*time.Second, func(page *Page, err error, out ActionData) {
require.Nil(t, err, "could not run page actions")
require.True(t, page.Page().MustElement("button").MustVisible(), "could not get button")
})
@@ -553,7 +556,7 @@ func TestActionWaitVisible(t *testing.T) {
}
t.Run("wait for an element being visible", func(t *testing.T) {
- testHeadlessSimpleResponse(t, response, actions, 2*time.Second, func(page *Page, err error, out map[string]string) {
+ testHeadlessSimpleResponse(t, response, actions, 2*time.Second, func(page *Page, err error, out ActionData) {
require.Nil(t, err, "could not run page actions")
page.Page().MustElement("button").MustVisible()
@@ -562,21 +565,82 @@ func TestActionWaitVisible(t *testing.T) {
t.Run("timeout because of element not visible", func(t *testing.T) {
// increased timeout from time.Second/2 to time.Second due to random fails (probably due to overhead and system)
- testHeadlessSimpleResponse(t, response, actions, time.Second, func(page *Page, err error, out map[string]string) {
+ testHeadlessSimpleResponse(t, response, actions, time.Second, func(page *Page, err error, out ActionData) {
require.Error(t, err)
require.Contains(t, err.Error(), "Element did not appear in the given amount of time")
})
})
}
-func testHeadlessSimpleResponse(t *testing.T, response string, actions []*Action, timeout time.Duration, assert func(page *Page, pageErr error, out map[string]string)) {
+func TestActionWaitDialog(t *testing.T) {
+ response := `
+
+ Nuclei Test Page
+
+
+
+
+ `
+
+ t.Run("Triggered", func(t *testing.T) {
+ actions := []*Action{
+ {
+ ActionType: ActionTypeHolder{ActionType: ActionNavigate},
+ Data: map[string]string{"url": "{{BaseURL}}/?script=alert%281%29"},
+ },
+ {
+ ActionType: ActionTypeHolder{ActionType: ActionWaitDialog},
+ Name: "test",
+ },
+ }
+
+ testHeadlessSimpleResponse(t, response, actions, 1*time.Second, func(page *Page, err error, out ActionData) {
+ require.Nil(t, err, "could not run page actions")
+
+ test, ok := out["test"].(bool)
+ require.True(t, ok, "could not assert test to bool")
+ require.True(t, test, "could not find test")
+ })
+ })
+
+ t.Run("Invalid", func(t *testing.T) {
+ actions := []*Action{
+ {
+ ActionType: ActionTypeHolder{ActionType: ActionNavigate},
+ Data: map[string]string{"url": "{{BaseURL}}/?script=foo"},
+ },
+ {
+ ActionType: ActionTypeHolder{ActionType: ActionWaitDialog},
+ Name: "test",
+ },
+ }
+
+ testHeadlessSimpleResponse(t, response, actions, 1*time.Second, func(page *Page, err error, out ActionData) {
+ require.Nil(t, err, "could not run page actions")
+
+ _, ok := out["test"].(bool)
+ require.False(t, ok, "output assertion is success")
+ })
+ })
+}
+
+func testHeadlessSimpleResponse(t *testing.T, response string, actions []*Action, timeout time.Duration, assert func(page *Page, pageErr error, out ActionData)) {
t.Helper()
testHeadless(t, actions, timeout, func(w http.ResponseWriter, r *http.Request) {
_, _ = fmt.Fprintln(w, response)
}, assert)
}
-func testHeadless(t *testing.T, actions []*Action, timeout time.Duration, handler func(w http.ResponseWriter, r *http.Request), assert func(page *Page, pageErr error, extractedData map[string]string)) {
+func testHeadless(t *testing.T, actions []*Action, timeout time.Duration, handler func(w http.ResponseWriter, r *http.Request), assert func(page *Page, pageErr error, extractedData ActionData)) {
t.Helper()
lfa := getBoolFromEnv("LOCAL_FILE_ACCESS", true)
diff --git a/pkg/protocols/headless/request.go b/pkg/protocols/headless/request.go
index c6fd28a5de..5f9b53174e 100644
--- a/pkg/protocols/headless/request.go
+++ b/pkg/protocols/headless/request.go
@@ -183,7 +183,13 @@ func (request *Request) executeRequestWithPayloads(input *contextargs.Context, p
responseBody, _ = html.HTML()
}
- outputEvent := request.responseToDSLMap(responseBody, out["header"], out["status_code"], reqBuilder.String(), input.MetaInput.Input, navigatedURL, page.DumpHistory())
+ header := out.GetOrDefault("header", "").(string)
+
+ // NOTE(dwisiswant0): `status_code` key should be an integer type.
+ // Ref: https://github.com/projectdiscovery/nuclei/pull/5545#discussion_r1721291013
+ statusCode := out.GetOrDefault("status_code", "").(string)
+
+ outputEvent := request.responseToDSLMap(responseBody, header, statusCode, reqBuilder.String(), input.MetaInput.Input, navigatedURL, page.DumpHistory())
// add response fields to template context and merge templatectx variables to output event
request.options.AddTemplateVars(input.MetaInput, request.Type(), request.ID, outputEvent)
if request.options.HasTemplateCtx(input.MetaInput) {
From cb05f55251691eaff8e8d821cfa4cfc0e85cbb40 Mon Sep 17 00:00:00 2001
From: ghost
Date: Mon, 2 Sep 2024 10:01:02 +0000
Subject: [PATCH 181/283] Auto Generate Syntax Docs + JSONSchema [Mon Sep 2
10:01:02 UTC 2024] :robot:
---
SYNTAX-REFERENCE.md | 2 ++
nuclei-jsonschema.json | 1 +
pkg/templates/templates_doc.go | 1 +
3 files changed, 4 insertions(+)
diff --git a/SYNTAX-REFERENCE.md b/SYNTAX-REFERENCE.md
index 28bb5bdf24..c757321494 100755
--- a/SYNTAX-REFERENCE.md
+++ b/SYNTAX-REFERENCE.md
@@ -3682,6 +3682,8 @@ Enum Values:
- waitevent
+ - dialog
+
- keyboard
- debug
diff --git a/nuclei-jsonschema.json b/nuclei-jsonschema.json
index cd8f8fa0b0..993a744502 100644
--- a/nuclei-jsonschema.json
+++ b/nuclei-jsonschema.json
@@ -256,6 +256,7 @@
"deleteheader",
"setbody",
"waitevent",
+ "waitdialog",
"keyboard",
"debug",
"sleep",
diff --git a/pkg/templates/templates_doc.go b/pkg/templates/templates_doc.go
index 5171e2a88f..b484dad565 100644
--- a/pkg/templates/templates_doc.go
+++ b/pkg/templates/templates_doc.go
@@ -1612,6 +1612,7 @@ func init() {
"deleteheader",
"setbody",
"waitevent",
+ "dialog",
"keyboard",
"debug",
"sleep",
From 0c4c0af4e9a6612d8dae300b08b58efdaf25f8a4 Mon Sep 17 00:00:00 2001
From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com>
Date: Mon, 2 Sep 2024 15:36:39 +0530
Subject: [PATCH 182/283] chore(deps): bump github.com/projectdiscovery/dsl
from 0.1.8 to 0.2.1 (#5587)
Bumps [github.com/projectdiscovery/dsl](https://github.com/projectdiscovery/dsl) from 0.1.8 to 0.2.1.
- [Release notes](https://github.com/projectdiscovery/dsl/releases)
- [Commits](https://github.com/projectdiscovery/dsl/compare/v0.1.8...v0.2.1)
---
updated-dependencies:
- dependency-name: github.com/projectdiscovery/dsl
dependency-type: direct:production
update-type: version-update:semver-minor
...
Signed-off-by: dependabot[bot]
Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com>
---
go.mod | 2 +-
go.sum | 4 ++--
2 files changed, 3 insertions(+), 3 deletions(-)
diff --git a/go.mod b/go.mod
index 7b3b596b96..075c1f0f76 100644
--- a/go.mod
+++ b/go.mod
@@ -81,7 +81,7 @@ require (
github.com/microsoft/go-mssqldb v1.6.0
github.com/ory/dockertest/v3 v3.10.0
github.com/praetorian-inc/fingerprintx v1.1.9
- github.com/projectdiscovery/dsl v0.1.8
+ github.com/projectdiscovery/dsl v0.2.1
github.com/projectdiscovery/fasttemplate v0.0.2
github.com/projectdiscovery/go-smb2 v0.0.0-20240129202741-052cc450c6cb
github.com/projectdiscovery/goflags v0.1.64
diff --git a/go.sum b/go.sum
index 2fedffa27a..fed643d2ab 100644
--- a/go.sum
+++ b/go.sum
@@ -832,8 +832,8 @@ github.com/projectdiscovery/cdncheck v1.1.0 h1:qDITidmJsejzpk3rMkauCh6sjI2GH9hW/
github.com/projectdiscovery/cdncheck v1.1.0/go.mod h1:sZ8U4MjHSsyaTVjBbYWHT1cwUVvUYwDX1W+WvWRicIc=
github.com/projectdiscovery/clistats v0.1.0 h1:b+LF1w0xhNd7cneKWMXb+/yUmF1n5szawxP4XGpmbxs=
github.com/projectdiscovery/clistats v0.1.0/go.mod h1:GJ2av0KnOvK0AISQnP8hyDclYIji1LVkx2l0pwnzAu4=
-github.com/projectdiscovery/dsl v0.1.8 h1:ulz+o097XsYgqP4QXaAQhVSkoeD2ZPWd29wX3CTodUA=
-github.com/projectdiscovery/dsl v0.1.8/go.mod h1:AYJS2WQ/q0smr2v4pEJTg4DPe6k56KFKR7UFXvzNz/4=
+github.com/projectdiscovery/dsl v0.2.1 h1:TK3KD4jsg4YbvY7WJqnz1QyH4AOvAwezeBFOX97Evgk=
+github.com/projectdiscovery/dsl v0.2.1/go.mod h1:IRQXsmi5/g1dDZ79//A9t2vrRtxm4frRSd5t8CZVSbI=
github.com/projectdiscovery/fastdialer v0.2.7 h1:x7bEf/kDyVOuG9kHfJLMhWO3h7J+6fMDgLWkGYL8EFg=
github.com/projectdiscovery/fastdialer v0.2.7/go.mod h1:7/BGdaw7sCekuBh6lLZqPuoO5N0Qaizwvex/PjO+V8o=
github.com/projectdiscovery/fasttemplate v0.0.2 h1:h2cISk5xDhlJEinlBQS6RRx0vOlOirB2y3Yu4PJzpiA=
From 04a61597130de20e730c928f68b086dfd67fbd0d Mon Sep 17 00:00:00 2001
From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com>
Date: Mon, 2 Sep 2024 15:46:16 +0530
Subject: [PATCH 183/283] chore(deps): bump
github.com/projectdiscovery/retryablehttp-go (#5591)
Bumps [github.com/projectdiscovery/retryablehttp-go](https://github.com/projectdiscovery/retryablehttp-go) from 1.0.74 to 1.0.76.
- [Release notes](https://github.com/projectdiscovery/retryablehttp-go/releases)
- [Commits](https://github.com/projectdiscovery/retryablehttp-go/compare/v1.0.74...v1.0.76)
---
updated-dependencies:
- dependency-name: github.com/projectdiscovery/retryablehttp-go
dependency-type: direct:production
update-type: version-update:semver-patch
...
Signed-off-by: dependabot[bot]
Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com>
---
go.mod | 2 +-
go.sum | 4 ++--
2 files changed, 3 insertions(+), 3 deletions(-)
diff --git a/go.mod b/go.mod
index 075c1f0f76..4a382aa7a4 100644
--- a/go.mod
+++ b/go.mod
@@ -25,7 +25,7 @@ require (
github.com/projectdiscovery/interactsh v1.2.0
github.com/projectdiscovery/rawhttp v0.1.60
github.com/projectdiscovery/retryabledns v1.0.73
- github.com/projectdiscovery/retryablehttp-go v1.0.74
+ github.com/projectdiscovery/retryablehttp-go v1.0.76
github.com/projectdiscovery/yamldoc-go v1.0.4
github.com/remeh/sizedwaitgroup v1.0.0
github.com/rs/xid v1.5.0
diff --git a/go.sum b/go.sum
index fed643d2ab..69164e9995 100644
--- a/go.sum
+++ b/go.sum
@@ -874,8 +874,8 @@ github.com/projectdiscovery/rdap v0.9.1-0.20221108103045-9865884d1917 h1:m03X4gB
github.com/projectdiscovery/rdap v0.9.1-0.20221108103045-9865884d1917/go.mod h1:JxXtZC9e195awe7EynrcnBJmFoad/BNDzW9mzFkK8Sg=
github.com/projectdiscovery/retryabledns v1.0.73 h1:CLqtgoI6+YVcUE2jnZPFVvwj59CHDE4HNImLf62sSuw=
github.com/projectdiscovery/retryabledns v1.0.73/go.mod h1:Wo5tQHKtx6XcYYhMypRg8dr0GGnqIb4CzoTSEw2lo7I=
-github.com/projectdiscovery/retryablehttp-go v1.0.74 h1:jXZsg3nNDuDjGNeL0+hiEx/dqUqPyZW1zsotWW1y70c=
-github.com/projectdiscovery/retryablehttp-go v1.0.74/go.mod h1:fVRxa2fVMbomaPiOFA05kM2WrjV8wPFHW7J84eenVmQ=
+github.com/projectdiscovery/retryablehttp-go v1.0.76 h1:2+4QpWnXUZFBkoeCEBXtUjSeBTdgKtaGwP8LRtKAC3g=
+github.com/projectdiscovery/retryablehttp-go v1.0.76/go.mod h1:0qIv7Ld8WWSKWYXMsNAWRYU5gsxnxpBza3SjDgt6nVc=
github.com/projectdiscovery/sarif v0.0.1 h1:C2Tyj0SGOKbCLgHrx83vaE6YkzXEVrMXYRGLkKCr/us=
github.com/projectdiscovery/sarif v0.0.1/go.mod h1:cEYlDu8amcPf6b9dSakcz2nNnJsoz4aR6peERwV+wuQ=
github.com/projectdiscovery/stringsutil v0.0.2 h1:uzmw3IVLJSMW1kEg8eCStG/cGbYYZAja8BH3LqqJXMA=
From 7bc257d7a57932816f0319de402f8ed8d71aa04a Mon Sep 17 00:00:00 2001
From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com>
Date: Mon, 2 Sep 2024 15:47:35 +0530
Subject: [PATCH 184/283] chore(deps): bump
github.com/projectdiscovery/ratelimit (#5588)
Bumps [github.com/projectdiscovery/ratelimit](https://github.com/projectdiscovery/ratelimit) from 0.0.49 to 0.0.53.
- [Release notes](https://github.com/projectdiscovery/ratelimit/releases)
- [Commits](https://github.com/projectdiscovery/ratelimit/compare/v0.0.49...v0.0.53)
---
updated-dependencies:
- dependency-name: github.com/projectdiscovery/ratelimit
dependency-type: direct:production
update-type: version-update:semver-patch
...
Signed-off-by: dependabot[bot]
Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com>
---
go.mod | 2 +-
go.sum | 4 ++--
2 files changed, 3 insertions(+), 3 deletions(-)
diff --git a/go.mod b/go.mod
index 4a382aa7a4..f3b56a408c 100644
--- a/go.mod
+++ b/go.mod
@@ -91,7 +91,7 @@ require (
github.com/projectdiscovery/httpx v1.6.7
github.com/projectdiscovery/mapcidr v1.1.34
github.com/projectdiscovery/n3iwf v0.0.0-20230523120440-b8cd232ff1f5
- github.com/projectdiscovery/ratelimit v0.0.49
+ github.com/projectdiscovery/ratelimit v0.0.53
github.com/projectdiscovery/rdap v0.9.1-0.20221108103045-9865884d1917
github.com/projectdiscovery/sarif v0.0.1
github.com/projectdiscovery/tlsx v1.1.7
diff --git a/go.sum b/go.sum
index 69164e9995..d50806fa64 100644
--- a/go.sum
+++ b/go.sum
@@ -866,8 +866,8 @@ github.com/projectdiscovery/n3iwf v0.0.0-20230523120440-b8cd232ff1f5 h1:L/e8z8yw
github.com/projectdiscovery/n3iwf v0.0.0-20230523120440-b8cd232ff1f5/go.mod h1:pGW2ncnTxTxHtP9wzcIJAB+3/NMp6IiuQWd2NK7K+oc=
github.com/projectdiscovery/networkpolicy v0.0.9 h1:IrlDoYZagNNO8y+7iZeHT8k5izE+nek7TdtvEBwCxqk=
github.com/projectdiscovery/networkpolicy v0.0.9/go.mod h1:XFJ2Lnv8BE/ziQCFjBHMsH1w6VmkPiQtk+NlBpdMU7M=
-github.com/projectdiscovery/ratelimit v0.0.49 h1:PYatMp8g5OuoFsZOA90e48nLd2vB6a4Tw0FZ8h9zqkQ=
-github.com/projectdiscovery/ratelimit v0.0.49/go.mod h1:Xi0LTMHg4HQlmCZFzRBIhRW6N+QW5RxQ8V/Qs+Vta4k=
+github.com/projectdiscovery/ratelimit v0.0.53 h1:1ci2zaVCtJ9daDi9IXjNwpDxl2InYq375role/xqUR4=
+github.com/projectdiscovery/ratelimit v0.0.53/go.mod h1:4kNYTL7xKLo1qMYHTREUCBqHyvykFF+XBWutTh9fkwg=
github.com/projectdiscovery/rawhttp v0.1.60 h1:L57kAATGRuCfOp8Fs0ly3j2wlip7R5pco2RVfwqh/II=
github.com/projectdiscovery/rawhttp v0.1.60/go.mod h1:M97w6lsphTHKeqLSio4Lw9K4KLO2riJMYAOqVMIZ3ck=
github.com/projectdiscovery/rdap v0.9.1-0.20221108103045-9865884d1917 h1:m03X4gBVSorSzvmm0bFa7gDV4QNSOWPL/fgZ4kTXBxk=
From 5bb730849ce124aca3a2e7177ce80d94217c493b Mon Sep 17 00:00:00 2001
From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com>
Date: Mon, 2 Sep 2024 15:55:34 +0530
Subject: [PATCH 185/283] chore(deps): bump github.com/projectdiscovery/httpx
from 1.6.7 to 1.6.8 (#5590)
Bumps [github.com/projectdiscovery/httpx](https://github.com/projectdiscovery/httpx) from 1.6.7 to 1.6.8.
- [Release notes](https://github.com/projectdiscovery/httpx/releases)
- [Changelog](https://github.com/projectdiscovery/httpx/blob/main/.goreleaser.yml)
- [Commits](https://github.com/projectdiscovery/httpx/compare/v1.6.7...v1.6.8)
---
updated-dependencies:
- dependency-name: github.com/projectdiscovery/httpx
dependency-type: direct:production
update-type: version-update:semver-patch
...
Signed-off-by: dependabot[bot]
Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com>
---
go.mod | 4 ++--
go.sum | 8 ++++----
2 files changed, 6 insertions(+), 6 deletions(-)
diff --git a/go.mod b/go.mod
index f3b56a408c..1a478943b8 100644
--- a/go.mod
+++ b/go.mod
@@ -23,7 +23,7 @@ require (
github.com/projectdiscovery/fastdialer v0.2.7
github.com/projectdiscovery/hmap v0.0.56
github.com/projectdiscovery/interactsh v1.2.0
- github.com/projectdiscovery/rawhttp v0.1.60
+ github.com/projectdiscovery/rawhttp v0.1.61
github.com/projectdiscovery/retryabledns v1.0.73
github.com/projectdiscovery/retryablehttp-go v1.0.76
github.com/projectdiscovery/yamldoc-go v1.0.4
@@ -88,7 +88,7 @@ require (
github.com/projectdiscovery/gologger v1.1.21
github.com/projectdiscovery/gostruct v0.0.2
github.com/projectdiscovery/gozero v0.0.2
- github.com/projectdiscovery/httpx v1.6.7
+ github.com/projectdiscovery/httpx v1.6.8
github.com/projectdiscovery/mapcidr v1.1.34
github.com/projectdiscovery/n3iwf v0.0.0-20230523120440-b8cd232ff1f5
github.com/projectdiscovery/ratelimit v0.0.53
diff --git a/go.sum b/go.sum
index d50806fa64..031530e8a4 100644
--- a/go.sum
+++ b/go.sum
@@ -852,8 +852,8 @@ github.com/projectdiscovery/gozero v0.0.2 h1:8fJeaCjxL9tpm33uG/RsCQs6HGM/NE6eA3c
github.com/projectdiscovery/gozero v0.0.2/go.mod h1:d8bZvDWW07LWNYWrwjZ4OO1I0cpkfqaysyDfSs9ibK8=
github.com/projectdiscovery/hmap v0.0.56 h1:QLIuFhCc7jrBH9WYpxNBDooSoH+/S8cCm7VO5rMt3lU=
github.com/projectdiscovery/hmap v0.0.56/go.mod h1:XyGKMO93nLOzpaTTCDBSKEGx/XWXjSpoyZJxT92Qd/Q=
-github.com/projectdiscovery/httpx v1.6.7 h1:luJAtQ+iKpiyKKDpcWw1/fF1MaRX6JZ4R3P+ol5Uikk=
-github.com/projectdiscovery/httpx v1.6.7/go.mod h1:rqcuexBrb4v36ZyX0BHcHaovTurOJn2P65Tdt9cYdfE=
+github.com/projectdiscovery/httpx v1.6.8 h1:k0Y5g3ue/7QbDP0+LykIxp/VhPDLfau3UEUyuxtP7qE=
+github.com/projectdiscovery/httpx v1.6.8/go.mod h1:7BIsDxyRwkBjthqFmEajXrA5f3yb4tlVfLmpNdf0ZXA=
github.com/projectdiscovery/interactsh v1.2.0 h1:Al6jHiR+Usl9egYJDLJaWNHOcH8Rugk8gWMasc8Cmw8=
github.com/projectdiscovery/interactsh v1.2.0/go.mod h1:Wxt0fnzxsfrAZQQlpVrf3xMatP4OXZaZbjuDkIQKdYY=
github.com/projectdiscovery/ldapserver v1.0.2-0.20240219154113-dcc758ebc0cb h1:MGtI4oE12ruWv11ZlPXXd7hl/uAaQZrFvrIDYDeVMd8=
@@ -868,8 +868,8 @@ github.com/projectdiscovery/networkpolicy v0.0.9 h1:IrlDoYZagNNO8y+7iZeHT8k5izE+
github.com/projectdiscovery/networkpolicy v0.0.9/go.mod h1:XFJ2Lnv8BE/ziQCFjBHMsH1w6VmkPiQtk+NlBpdMU7M=
github.com/projectdiscovery/ratelimit v0.0.53 h1:1ci2zaVCtJ9daDi9IXjNwpDxl2InYq375role/xqUR4=
github.com/projectdiscovery/ratelimit v0.0.53/go.mod h1:4kNYTL7xKLo1qMYHTREUCBqHyvykFF+XBWutTh9fkwg=
-github.com/projectdiscovery/rawhttp v0.1.60 h1:L57kAATGRuCfOp8Fs0ly3j2wlip7R5pco2RVfwqh/II=
-github.com/projectdiscovery/rawhttp v0.1.60/go.mod h1:M97w6lsphTHKeqLSio4Lw9K4KLO2riJMYAOqVMIZ3ck=
+github.com/projectdiscovery/rawhttp v0.1.61 h1:EbskCj6kkDSG31sO5zEUFTqHp9ltccG1DdcI+MCdahQ=
+github.com/projectdiscovery/rawhttp v0.1.61/go.mod h1:5XmDAKph9pLVnh87zjL+vXDpfG5W8Gz5N2BAtYTkuVU=
github.com/projectdiscovery/rdap v0.9.1-0.20221108103045-9865884d1917 h1:m03X4gBVSorSzvmm0bFa7gDV4QNSOWPL/fgZ4kTXBxk=
github.com/projectdiscovery/rdap v0.9.1-0.20221108103045-9865884d1917/go.mod h1:JxXtZC9e195awe7EynrcnBJmFoad/BNDzW9mzFkK8Sg=
github.com/projectdiscovery/retryabledns v1.0.73 h1:CLqtgoI6+YVcUE2jnZPFVvwj59CHDE4HNImLf62sSuw=
From 042b33de3dec66ab92c392e7450cc25c0b76859b Mon Sep 17 00:00:00 2001
From: sandeep <8293321+ehsandeep@users.noreply.github.com>
Date: Mon, 2 Sep 2024 17:38:48 +0530
Subject: [PATCH 186/283] version update
---
pkg/catalog/config/constants.go | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/pkg/catalog/config/constants.go b/pkg/catalog/config/constants.go
index cd0e5673c7..ff54ce9bc0 100644
--- a/pkg/catalog/config/constants.go
+++ b/pkg/catalog/config/constants.go
@@ -31,7 +31,7 @@ const (
CLIConfigFileName = "config.yaml"
ReportingConfigFilename = "reporting-config.yaml"
// Version is the current version of nuclei
- Version = `v3.3.1`
+ Version = `v3.3.2`
// Directory Names of custom templates
CustomS3TemplatesDirName = "s3"
CustomGitHubTemplatesDirName = "github"
From f30d0b3b8dc817a0416fb83bb3529a246132aeb0 Mon Sep 17 00:00:00 2001
From: Ramana Reddy <90540245+RamanaReddy0M@users.noreply.github.com>
Date: Fri, 6 Sep 2024 23:49:55 +0530
Subject: [PATCH 187/283] fix: mysql connection with special characters in
password (#5604)
---
pkg/js/libs/mysql/mysql_private.go | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/pkg/js/libs/mysql/mysql_private.go b/pkg/js/libs/mysql/mysql_private.go
index aa89df2600..42e76a5579 100644
--- a/pkg/js/libs/mysql/mysql_private.go
+++ b/pkg/js/libs/mysql/mysql_private.go
@@ -53,7 +53,7 @@ func BuildDSN(opts MySQLOptions) (string, error) {
}
target := net.JoinHostPort(opts.Host, fmt.Sprintf("%d", opts.Port))
var dsn strings.Builder
- dsn.WriteString(fmt.Sprintf("%v:%v", url.QueryEscape(opts.Username), url.QueryEscape(opts.Password)))
+ dsn.WriteString(fmt.Sprintf("%v:%v", url.QueryEscape(opts.Username), opts.Password))
dsn.WriteString("@")
dsn.WriteString(fmt.Sprintf("%v(%v)", opts.Protocol, target))
if opts.DbName != "" {
From 2b5a8f5b63cded0019a1c54764f215dc422e45df Mon Sep 17 00:00:00 2001
From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com>
Date: Fri, 6 Sep 2024 23:51:56 +0530
Subject: [PATCH 188/283] chore(deps): bump github.com/opencontainers/runc from
1.1.12 to 1.1.14 (#5596)
Bumps [github.com/opencontainers/runc](https://github.com/opencontainers/runc) from 1.1.12 to 1.1.14.
- [Release notes](https://github.com/opencontainers/runc/releases)
- [Changelog](https://github.com/opencontainers/runc/blob/main/CHANGELOG.md)
- [Commits](https://github.com/opencontainers/runc/compare/v1.1.12...v1.1.14)
---
updated-dependencies:
- dependency-name: github.com/opencontainers/runc
dependency-type: indirect
...
Signed-off-by: dependabot[bot]
Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com>
---
go.mod | 2 +-
go.sum | 4 ++--
2 files changed, 3 insertions(+), 3 deletions(-)
diff --git a/go.mod b/go.mod
index 1a478943b8..ec9faea29d 100644
--- a/go.mod
+++ b/go.mod
@@ -195,7 +195,7 @@ require (
github.com/muesli/termenv v0.15.1 // indirect
github.com/opencontainers/go-digest v1.0.0 // indirect
github.com/opencontainers/image-spec v1.0.2 // indirect
- github.com/opencontainers/runc v1.1.12 // indirect
+ github.com/opencontainers/runc v1.1.14 // indirect
github.com/pelletier/go-toml/v2 v2.0.8 // indirect
github.com/perimeterx/marshmallow v1.1.5 // indirect
github.com/pierrec/lz4/v4 v4.1.21 // indirect
diff --git a/go.sum b/go.sum
index 031530e8a4..3fca3d09db 100644
--- a/go.sum
+++ b/go.sum
@@ -781,8 +781,8 @@ github.com/opencontainers/go-digest v1.0.0 h1:apOUWs51W5PlhuyGyz9FCeeBIOUDA/6nW8
github.com/opencontainers/go-digest v1.0.0/go.mod h1:0JzlMkj0TRzQZfJkVvzbP0HBR3IKzErnv2BNG4W4MAM=
github.com/opencontainers/image-spec v1.0.2 h1:9yCKha/T5XdGtO0q9Q9a6T5NUCsTn/DrBg0D7ufOcFM=
github.com/opencontainers/image-spec v1.0.2/go.mod h1:BtxoFyWECRxE4U/7sNtV5W15zMzWCbyJoFRP3s7yZA0=
-github.com/opencontainers/runc v1.1.12 h1:BOIssBaW1La0/qbNZHXOOa71dZfZEQOzW7dqQf3phss=
-github.com/opencontainers/runc v1.1.12/go.mod h1:S+lQwSfncpBha7XTy/5lBwWgm5+y5Ma/O44Ekby9FK8=
+github.com/opencontainers/runc v1.1.14 h1:rgSuzbmgz5DUJjeSnw337TxDbRuqjs6iqQck/2weR6w=
+github.com/opencontainers/runc v1.1.14/go.mod h1:E4C2z+7BxR7GHXp0hAY53mek+x49X1LjPNeMTfRGvOA=
github.com/opentracing-contrib/go-observer v0.0.0-20170622124052-a52f23424492/go.mod h1:Ngi6UdF0k5OKD5t5wlmGhe/EDKPoUM3BXZSSfIuJbis=
github.com/opentracing/basictracer-go v1.0.0/go.mod h1:QfBfYuafItcjQuMwinw9GhYKwFXS9KnPs5lxoYwgW74=
github.com/opentracing/opentracing-go v1.0.2/go.mod h1:UkNAQd3GIcIGf0SeVgPpRdFStlNbqXla1AfSYxPUl2o=
From cc332979aabb803d325540053cc27f8b0ab19b60 Mon Sep 17 00:00:00 2001
From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com>
Date: Mon, 9 Sep 2024 05:21:46 +0000
Subject: [PATCH 189/283] chore(deps): bump
github.com/projectdiscovery/clistats
Bumps [github.com/projectdiscovery/clistats](https://github.com/projectdiscovery/clistats) from 0.1.0 to 0.1.1.
- [Release notes](https://github.com/projectdiscovery/clistats/releases)
- [Commits](https://github.com/projectdiscovery/clistats/compare/v0.1.0...v0.1.1)
---
updated-dependencies:
- dependency-name: github.com/projectdiscovery/clistats
dependency-type: direct:production
update-type: version-update:semver-patch
...
Signed-off-by: dependabot[bot]
---
go.mod | 4 ++--
go.sum | 8 ++++----
2 files changed, 6 insertions(+), 6 deletions(-)
diff --git a/go.mod b/go.mod
index ec9faea29d..a1dc198bb2 100644
--- a/go.mod
+++ b/go.mod
@@ -19,7 +19,7 @@ require (
github.com/miekg/dns v1.1.59
github.com/olekukonko/tablewriter v0.0.5
github.com/pkg/errors v0.9.1
- github.com/projectdiscovery/clistats v0.1.0
+ github.com/projectdiscovery/clistats v0.1.1
github.com/projectdiscovery/fastdialer v0.2.7
github.com/projectdiscovery/hmap v0.0.56
github.com/projectdiscovery/interactsh v1.2.0
@@ -203,7 +203,7 @@ require (
github.com/pkg/browser v0.0.0-20240102092130-5ac0b6a4141c // indirect
github.com/projectdiscovery/asnmap v1.1.1 // indirect
github.com/projectdiscovery/cdncheck v1.1.0 // indirect
- github.com/projectdiscovery/freeport v0.0.5 // indirect
+ github.com/projectdiscovery/freeport v0.0.6 // indirect
github.com/projectdiscovery/ldapserver v1.0.2-0.20240219154113-dcc758ebc0cb // indirect
github.com/projectdiscovery/machineid v0.0.0-20240226150047-2e2c51e35983 // indirect
github.com/refraction-networking/utls v1.6.7 // indirect
diff --git a/go.sum b/go.sum
index 3fca3d09db..aa6318788d 100644
--- a/go.sum
+++ b/go.sum
@@ -830,16 +830,16 @@ github.com/projectdiscovery/blackrock v0.0.1 h1:lHQqhaaEFjgf5WkuItbpeCZv2DUIE45k
github.com/projectdiscovery/blackrock v0.0.1/go.mod h1:ANUtjDfaVrqB453bzToU+YB4cUbvBRpLvEwoWIwlTss=
github.com/projectdiscovery/cdncheck v1.1.0 h1:qDITidmJsejzpk3rMkauCh6sjI2GH9hW/snk0cQ3kXE=
github.com/projectdiscovery/cdncheck v1.1.0/go.mod h1:sZ8U4MjHSsyaTVjBbYWHT1cwUVvUYwDX1W+WvWRicIc=
-github.com/projectdiscovery/clistats v0.1.0 h1:b+LF1w0xhNd7cneKWMXb+/yUmF1n5szawxP4XGpmbxs=
-github.com/projectdiscovery/clistats v0.1.0/go.mod h1:GJ2av0KnOvK0AISQnP8hyDclYIji1LVkx2l0pwnzAu4=
+github.com/projectdiscovery/clistats v0.1.1 h1:8mwbdbwTU4aT88TJvwIzTpiNeow3XnAB72JIg66c8wE=
+github.com/projectdiscovery/clistats v0.1.1/go.mod h1:4LtTC9Oy//RiuT1+76MfTg8Hqs7FQp1JIGBM3nHK6a0=
github.com/projectdiscovery/dsl v0.2.1 h1:TK3KD4jsg4YbvY7WJqnz1QyH4AOvAwezeBFOX97Evgk=
github.com/projectdiscovery/dsl v0.2.1/go.mod h1:IRQXsmi5/g1dDZ79//A9t2vrRtxm4frRSd5t8CZVSbI=
github.com/projectdiscovery/fastdialer v0.2.7 h1:x7bEf/kDyVOuG9kHfJLMhWO3h7J+6fMDgLWkGYL8EFg=
github.com/projectdiscovery/fastdialer v0.2.7/go.mod h1:7/BGdaw7sCekuBh6lLZqPuoO5N0Qaizwvex/PjO+V8o=
github.com/projectdiscovery/fasttemplate v0.0.2 h1:h2cISk5xDhlJEinlBQS6RRx0vOlOirB2y3Yu4PJzpiA=
github.com/projectdiscovery/fasttemplate v0.0.2/go.mod h1:XYWWVMxnItd+r0GbjA1GCsUopMw1/XusuQxdyAIHMCw=
-github.com/projectdiscovery/freeport v0.0.5 h1:jnd3Oqsl4S8n0KuFkE5Hm8WGDP24ITBvmyw5pFTHS8Q=
-github.com/projectdiscovery/freeport v0.0.5/go.mod h1:PY0bxSJ34HVy67LHIeF3uIutiCSDwOqKD8ruBkdiCwE=
+github.com/projectdiscovery/freeport v0.0.6 h1:ROqzuXN8JPqkGdBueb3ah691nS2g2p7r3/3x2E33GbI=
+github.com/projectdiscovery/freeport v0.0.6/go.mod h1:T2kIy+WrbyxBIhI8V3Y9aeNGnuhnM8tEUSK/cm9GjAg=
github.com/projectdiscovery/go-smb2 v0.0.0-20240129202741-052cc450c6cb h1:rutG906Drtbpz4DwU5mhGIeOhRcktDH4cGQitGUMAsg=
github.com/projectdiscovery/go-smb2 v0.0.0-20240129202741-052cc450c6cb/go.mod h1:FLjF1DmZ+POoGEiIQdWuYVwS++C/GwpX8YaCsTSm1RY=
github.com/projectdiscovery/goflags v0.1.64 h1:FDfwdt9N97Hi8OuhbkDlKtVttpc/CRMIWQVa08VsHsI=
From 6b0ba0f4f3f5f67eb81746fabdbb26f6449f62fc Mon Sep 17 00:00:00 2001
From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com>
Date: Mon, 9 Sep 2024 05:21:53 +0000
Subject: [PATCH 190/283] chore(deps): bump github.com/projectdiscovery/rawhttp
Bumps [github.com/projectdiscovery/rawhttp](https://github.com/projectdiscovery/rawhttp) from 0.1.61 to 0.1.65.
- [Release notes](https://github.com/projectdiscovery/rawhttp/releases)
- [Commits](https://github.com/projectdiscovery/rawhttp/compare/v0.1.61...v0.1.65)
---
updated-dependencies:
- dependency-name: github.com/projectdiscovery/rawhttp
dependency-type: direct:production
update-type: version-update:semver-patch
...
Signed-off-by: dependabot[bot]
---
go.mod | 8 ++++----
go.sum | 16 ++++++++--------
2 files changed, 12 insertions(+), 12 deletions(-)
diff --git a/go.mod b/go.mod
index ec9faea29d..da9ce168a9 100644
--- a/go.mod
+++ b/go.mod
@@ -21,10 +21,10 @@ require (
github.com/pkg/errors v0.9.1
github.com/projectdiscovery/clistats v0.1.0
github.com/projectdiscovery/fastdialer v0.2.7
- github.com/projectdiscovery/hmap v0.0.56
+ github.com/projectdiscovery/hmap v0.0.57
github.com/projectdiscovery/interactsh v1.2.0
- github.com/projectdiscovery/rawhttp v0.1.61
- github.com/projectdiscovery/retryabledns v1.0.73
+ github.com/projectdiscovery/rawhttp v0.1.65
+ github.com/projectdiscovery/retryabledns v1.0.74
github.com/projectdiscovery/retryablehttp-go v1.0.76
github.com/projectdiscovery/yamldoc-go v1.0.4
github.com/remeh/sizedwaitgroup v1.0.0
@@ -97,7 +97,7 @@ require (
github.com/projectdiscovery/tlsx v1.1.7
github.com/projectdiscovery/uncover v1.0.9
github.com/projectdiscovery/useragent v0.0.65
- github.com/projectdiscovery/utils v0.2.7
+ github.com/projectdiscovery/utils v0.2.8
github.com/projectdiscovery/wappalyzergo v0.1.14
github.com/redis/go-redis/v9 v9.1.0
github.com/seh-msft/burpxml v1.0.1
diff --git a/go.sum b/go.sum
index 3fca3d09db..28404585a6 100644
--- a/go.sum
+++ b/go.sum
@@ -850,8 +850,8 @@ github.com/projectdiscovery/gostruct v0.0.2 h1:s8gP8ApugGM4go1pA+sVlPDXaWqNP5BBD
github.com/projectdiscovery/gostruct v0.0.2/go.mod h1:H86peL4HKwMXcQQtEa6lmC8FuD9XFt6gkNR0B/Mu5PE=
github.com/projectdiscovery/gozero v0.0.2 h1:8fJeaCjxL9tpm33uG/RsCQs6HGM/NE6eA3cjkilRQ+E=
github.com/projectdiscovery/gozero v0.0.2/go.mod h1:d8bZvDWW07LWNYWrwjZ4OO1I0cpkfqaysyDfSs9ibK8=
-github.com/projectdiscovery/hmap v0.0.56 h1:QLIuFhCc7jrBH9WYpxNBDooSoH+/S8cCm7VO5rMt3lU=
-github.com/projectdiscovery/hmap v0.0.56/go.mod h1:XyGKMO93nLOzpaTTCDBSKEGx/XWXjSpoyZJxT92Qd/Q=
+github.com/projectdiscovery/hmap v0.0.57 h1:n45T9Me/fL0fqdVZ+Pi/eVizBguNRCleThqxgzNPLrA=
+github.com/projectdiscovery/hmap v0.0.57/go.mod h1:aa7egkpPIBukf5tis936KSddiXfEvWUKYLzrtV3CH8M=
github.com/projectdiscovery/httpx v1.6.8 h1:k0Y5g3ue/7QbDP0+LykIxp/VhPDLfau3UEUyuxtP7qE=
github.com/projectdiscovery/httpx v1.6.8/go.mod h1:7BIsDxyRwkBjthqFmEajXrA5f3yb4tlVfLmpNdf0ZXA=
github.com/projectdiscovery/interactsh v1.2.0 h1:Al6jHiR+Usl9egYJDLJaWNHOcH8Rugk8gWMasc8Cmw8=
@@ -868,12 +868,12 @@ github.com/projectdiscovery/networkpolicy v0.0.9 h1:IrlDoYZagNNO8y+7iZeHT8k5izE+
github.com/projectdiscovery/networkpolicy v0.0.9/go.mod h1:XFJ2Lnv8BE/ziQCFjBHMsH1w6VmkPiQtk+NlBpdMU7M=
github.com/projectdiscovery/ratelimit v0.0.53 h1:1ci2zaVCtJ9daDi9IXjNwpDxl2InYq375role/xqUR4=
github.com/projectdiscovery/ratelimit v0.0.53/go.mod h1:4kNYTL7xKLo1qMYHTREUCBqHyvykFF+XBWutTh9fkwg=
-github.com/projectdiscovery/rawhttp v0.1.61 h1:EbskCj6kkDSG31sO5zEUFTqHp9ltccG1DdcI+MCdahQ=
-github.com/projectdiscovery/rawhttp v0.1.61/go.mod h1:5XmDAKph9pLVnh87zjL+vXDpfG5W8Gz5N2BAtYTkuVU=
+github.com/projectdiscovery/rawhttp v0.1.65 h1:/SGp+UbFBSDquEHgWGA8VRH3kYiM9uMFQjfLfXBKOfY=
+github.com/projectdiscovery/rawhttp v0.1.65/go.mod h1:X7kvZ1uTzdSHc5bb68qmmpit6DGuqc1wXQLfFNfRJq0=
github.com/projectdiscovery/rdap v0.9.1-0.20221108103045-9865884d1917 h1:m03X4gBVSorSzvmm0bFa7gDV4QNSOWPL/fgZ4kTXBxk=
github.com/projectdiscovery/rdap v0.9.1-0.20221108103045-9865884d1917/go.mod h1:JxXtZC9e195awe7EynrcnBJmFoad/BNDzW9mzFkK8Sg=
-github.com/projectdiscovery/retryabledns v1.0.73 h1:CLqtgoI6+YVcUE2jnZPFVvwj59CHDE4HNImLf62sSuw=
-github.com/projectdiscovery/retryabledns v1.0.73/go.mod h1:Wo5tQHKtx6XcYYhMypRg8dr0GGnqIb4CzoTSEw2lo7I=
+github.com/projectdiscovery/retryabledns v1.0.74 h1:4NtSjnVdPWT1NuabOMf5vjUoky3X1DwJN6iFanrtQnk=
+github.com/projectdiscovery/retryabledns v1.0.74/go.mod h1:GRBpKmWIds0UVeDpqSek2Q8RITuuIZWD5wHL4cN5geg=
github.com/projectdiscovery/retryablehttp-go v1.0.76 h1:2+4QpWnXUZFBkoeCEBXtUjSeBTdgKtaGwP8LRtKAC3g=
github.com/projectdiscovery/retryablehttp-go v1.0.76/go.mod h1:0qIv7Ld8WWSKWYXMsNAWRYU5gsxnxpBza3SjDgt6nVc=
github.com/projectdiscovery/sarif v0.0.1 h1:C2Tyj0SGOKbCLgHrx83vaE6YkzXEVrMXYRGLkKCr/us=
@@ -886,8 +886,8 @@ github.com/projectdiscovery/uncover v1.0.9 h1:s5RbkD/V4r8QcPkys4gTTqMuRSgXq0Jpre
github.com/projectdiscovery/uncover v1.0.9/go.mod h1:2PUF3SpB5QNIJ8epaB2xbRzkPaxEAWRDm3Ir2ijt81U=
github.com/projectdiscovery/useragent v0.0.65 h1:x78ZwWdqpzokOHxLITUXvq+ljkTKc19z3ILGtoV1N70=
github.com/projectdiscovery/useragent v0.0.65/go.mod h1:deOP8YLJU6SCzM8k+K8PjkcOF4Ux0spqyO4ODZGIT4A=
-github.com/projectdiscovery/utils v0.2.7 h1:XWdz7SscL++jqsnQ9ecHzSZE0RK33tyPcnqcXw+vmKs=
-github.com/projectdiscovery/utils v0.2.7/go.mod h1:N0N7tbdNFPegd9NpJ3onCPClaBrERcOIB88yww6UCF8=
+github.com/projectdiscovery/utils v0.2.8 h1:++NcCJ+lXEfNBHKBs6q+cWa8JrVS8cYdGcW9jOgZebI=
+github.com/projectdiscovery/utils v0.2.8/go.mod h1:UYJ8GKbZaezPFRT/cOk01LreQZ1QK0fko1EUnSUiSGU=
github.com/projectdiscovery/wappalyzergo v0.1.14 h1:nt1IM4RUmqeymsXk4h6BsZbKDoS2hjFvPkT2GaI1rz4=
github.com/projectdiscovery/wappalyzergo v0.1.14/go.mod h1:/hzgxkBFTMe2wDbA93nFfoMjULw7/vIZ9QPSAnCgUa8=
github.com/projectdiscovery/yamldoc-go v1.0.4 h1:eZoESapnMw6WAHiVgRwNqvbJEfNHEH148uthhFbG5jE=
From 7f4cdf61965d2d49d58fd46ec327e1a7a8d2c23b Mon Sep 17 00:00:00 2001
From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com>
Date: Mon, 9 Sep 2024 05:21:57 +0000
Subject: [PATCH 191/283] chore(deps): bump
github.com/projectdiscovery/gologger
Bumps [github.com/projectdiscovery/gologger](https://github.com/projectdiscovery/gologger) from 1.1.21 to 1.1.23.
- [Release notes](https://github.com/projectdiscovery/gologger/releases)
- [Commits](https://github.com/projectdiscovery/gologger/compare/v1.1.21...v1.1.23)
---
updated-dependencies:
- dependency-name: github.com/projectdiscovery/gologger
dependency-type: direct:production
update-type: version-update:semver-patch
...
Signed-off-by: dependabot[bot]
---
go.mod | 4 ++--
go.sum | 8 ++++----
2 files changed, 6 insertions(+), 6 deletions(-)
diff --git a/go.mod b/go.mod
index ec9faea29d..25f912424a 100644
--- a/go.mod
+++ b/go.mod
@@ -85,7 +85,7 @@ require (
github.com/projectdiscovery/fasttemplate v0.0.2
github.com/projectdiscovery/go-smb2 v0.0.0-20240129202741-052cc450c6cb
github.com/projectdiscovery/goflags v0.1.64
- github.com/projectdiscovery/gologger v1.1.21
+ github.com/projectdiscovery/gologger v1.1.23
github.com/projectdiscovery/gostruct v0.0.2
github.com/projectdiscovery/gozero v0.0.2
github.com/projectdiscovery/httpx v1.6.8
@@ -97,7 +97,7 @@ require (
github.com/projectdiscovery/tlsx v1.1.7
github.com/projectdiscovery/uncover v1.0.9
github.com/projectdiscovery/useragent v0.0.65
- github.com/projectdiscovery/utils v0.2.7
+ github.com/projectdiscovery/utils v0.2.8
github.com/projectdiscovery/wappalyzergo v0.1.14
github.com/redis/go-redis/v9 v9.1.0
github.com/seh-msft/burpxml v1.0.1
diff --git a/go.sum b/go.sum
index 3fca3d09db..b8d4343fd6 100644
--- a/go.sum
+++ b/go.sum
@@ -844,8 +844,8 @@ github.com/projectdiscovery/go-smb2 v0.0.0-20240129202741-052cc450c6cb h1:rutG90
github.com/projectdiscovery/go-smb2 v0.0.0-20240129202741-052cc450c6cb/go.mod h1:FLjF1DmZ+POoGEiIQdWuYVwS++C/GwpX8YaCsTSm1RY=
github.com/projectdiscovery/goflags v0.1.64 h1:FDfwdt9N97Hi8OuhbkDlKtVttpc/CRMIWQVa08VsHsI=
github.com/projectdiscovery/goflags v0.1.64/go.mod h1:3FyHIVQtnycNOc1LE3O1jj/XR5XuMdF9QfHd0ujhnX4=
-github.com/projectdiscovery/gologger v1.1.21 h1:FD9MGE9adY/y2IxDpQgPZIvZAkQIx3bvcuY+d1FLQUo=
-github.com/projectdiscovery/gologger v1.1.21/go.mod h1:eyvlcHiSuzJQtwmh+fcfwZVaqVgSGtEP1oL1gJFgP3c=
+github.com/projectdiscovery/gologger v1.1.23 h1:u9ksRgkWuUoaOqkk0Z1FngWriXH0Bnt73yYwFGmZo8E=
+github.com/projectdiscovery/gologger v1.1.23/go.mod h1:zcFt3u4U0kmhXou7cTXWwwu+yqVPX3GSABD1hqU0Ur8=
github.com/projectdiscovery/gostruct v0.0.2 h1:s8gP8ApugGM4go1pA+sVlPDXaWqNP5BBDDSv7VEdG1M=
github.com/projectdiscovery/gostruct v0.0.2/go.mod h1:H86peL4HKwMXcQQtEa6lmC8FuD9XFt6gkNR0B/Mu5PE=
github.com/projectdiscovery/gozero v0.0.2 h1:8fJeaCjxL9tpm33uG/RsCQs6HGM/NE6eA3cjkilRQ+E=
@@ -886,8 +886,8 @@ github.com/projectdiscovery/uncover v1.0.9 h1:s5RbkD/V4r8QcPkys4gTTqMuRSgXq0Jpre
github.com/projectdiscovery/uncover v1.0.9/go.mod h1:2PUF3SpB5QNIJ8epaB2xbRzkPaxEAWRDm3Ir2ijt81U=
github.com/projectdiscovery/useragent v0.0.65 h1:x78ZwWdqpzokOHxLITUXvq+ljkTKc19z3ILGtoV1N70=
github.com/projectdiscovery/useragent v0.0.65/go.mod h1:deOP8YLJU6SCzM8k+K8PjkcOF4Ux0spqyO4ODZGIT4A=
-github.com/projectdiscovery/utils v0.2.7 h1:XWdz7SscL++jqsnQ9ecHzSZE0RK33tyPcnqcXw+vmKs=
-github.com/projectdiscovery/utils v0.2.7/go.mod h1:N0N7tbdNFPegd9NpJ3onCPClaBrERcOIB88yww6UCF8=
+github.com/projectdiscovery/utils v0.2.8 h1:++NcCJ+lXEfNBHKBs6q+cWa8JrVS8cYdGcW9jOgZebI=
+github.com/projectdiscovery/utils v0.2.8/go.mod h1:UYJ8GKbZaezPFRT/cOk01LreQZ1QK0fko1EUnSUiSGU=
github.com/projectdiscovery/wappalyzergo v0.1.14 h1:nt1IM4RUmqeymsXk4h6BsZbKDoS2hjFvPkT2GaI1rz4=
github.com/projectdiscovery/wappalyzergo v0.1.14/go.mod h1:/hzgxkBFTMe2wDbA93nFfoMjULw7/vIZ9QPSAnCgUa8=
github.com/projectdiscovery/yamldoc-go v1.0.4 h1:eZoESapnMw6WAHiVgRwNqvbJEfNHEH148uthhFbG5jE=
From fcdead9157a23cee596e9dc90c4488e222c05e47 Mon Sep 17 00:00:00 2001
From: Dwi Siswanto
Date: Mon, 9 Sep 2024 20:54:49 +0700
Subject: [PATCH 192/283] chore: bump `go-ci` to `v1.0.3` (#5611)
Signed-off-by: Dwi Siswanto
---
go.mod | 2 +-
go.sum | 2 ++
2 files changed, 3 insertions(+), 1 deletion(-)
diff --git a/go.mod b/go.mod
index 16cc0543d3..e24efbbe45 100644
--- a/go.mod
+++ b/go.mod
@@ -72,7 +72,7 @@ require (
github.com/go-sql-driver/mysql v1.7.1
github.com/h2non/filetype v1.1.3
github.com/invopop/yaml v0.3.1
- github.com/kitabisa/go-ci v1.0.2
+ github.com/kitabisa/go-ci v1.0.3
github.com/labstack/echo/v4 v4.10.2
github.com/leslie-qiwa/flat v0.0.0-20230424180412-f9d1cf014baa
github.com/lib/pq v1.10.9
diff --git a/go.sum b/go.sum
index 3ef1939612..77b6bdfd3b 100644
--- a/go.sum
+++ b/go.sum
@@ -627,6 +627,8 @@ github.com/kisielk/errcheck v1.5.0/go.mod h1:pFxgyoBC7bSaBwPgfKdkLd5X25qrDl4LWUI
github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck=
github.com/kitabisa/go-ci v1.0.2 h1:rqHf8KEbQOxVb998TbqGRo70Z7ol44io7/jLYJUvKp8=
github.com/kitabisa/go-ci v1.0.2/go.mod h1:e3wBSzaJbcifXrr/Gw2ZBLn44MmeqP5WySwXyHlCK/U=
+github.com/kitabisa/go-ci v1.0.3 h1:JmIUIvcercRQc/9x/v02ydCCqU4MadSHaNaOF8T2pGA=
+github.com/kitabisa/go-ci v1.0.3/go.mod h1:e3wBSzaJbcifXrr/Gw2ZBLn44MmeqP5WySwXyHlCK/U=
github.com/klauspost/compress v1.4.1/go.mod h1:RyIbtBH6LamlWaDj8nUwkbUhJ87Yi3uG0guNDohfE1A=
github.com/klauspost/compress v1.11.4/go.mod h1:aoV0uJVorq1K+umq18yTdKaF57EivdYsUV+/s2qKfXs=
github.com/klauspost/compress v1.17.8 h1:YcnTYrq7MikUT7k0Yb5eceMmALQPYBW/Xltxn0NAMnU=
From 59e69b561da5ed518e39abb220975a69a5b90654 Mon Sep 17 00:00:00 2001
From: Ice3man
Date: Thu, 12 Sep 2024 15:34:50 +0530
Subject: [PATCH 193/283] feat: added linear issue tracker support to nuclei
(#5601)
* feat: added linear issue tracker support to nuclei
* misc
* feat: fixed unmarshal issues
* added linear config
---------
Co-authored-by: sandeep <8293321+ehsandeep@users.noreply.github.com>
---
cmd/nuclei/issue-tracker-config.yaml | 21 +-
go.mod | 1 +
go.sum | 2 +
pkg/reporting/options.go | 3 +
pkg/reporting/reporting.go | 11 +
.../trackers/linear/jsonutil/jsonutil.go | 312 ++++++++++++++
pkg/reporting/trackers/linear/linear.go | 404 ++++++++++++++++++
7 files changed, 753 insertions(+), 1 deletion(-)
create mode 100644 pkg/reporting/trackers/linear/jsonutil/jsonutil.go
create mode 100644 pkg/reporting/trackers/linear/linear.go
diff --git a/cmd/nuclei/issue-tracker-config.yaml b/cmd/nuclei/issue-tracker-config.yaml
index 51778eb0ad..b7e0e6dafc 100644
--- a/cmd/nuclei/issue-tracker-config.yaml
+++ b/cmd/nuclei/issue-tracker-config.yaml
@@ -142,4 +142,23 @@
# # Username for the elasticsearch instance
# username: test
# # Password is the password for elasticsearch instance
-# password: test
\ No newline at end of file
+# password: test
+#linear:
+# # api-key is the API key for the linear account
+# api-key: ""
+# # allow-list sets a tracker level filter to only create issues for templates with
+# # these severity labels or tags (does not affect exporters. set those globally)
+# deny-list:
+# severity: critical
+# # deny-list sets a tracker level filter to never create issues for templates with
+# # these severity labels or tags (does not affect exporters. set those globally)
+# deny-list:
+# severity: low
+# # team-id is the ID of the team in Linear
+# team-id: ""
+# # project-id is the ID of the project in Linear
+# project-id: ""
+# # duplicate-issue-check flag to enable duplicate tracking issue check
+# duplicate-issue-check: false
+# # open-state-id is the ID of the open state in Linear
+# open-state-id: ""
diff --git a/go.mod b/go.mod
index e24efbbe45..7b5e247bf7 100644
--- a/go.mod
+++ b/go.mod
@@ -101,6 +101,7 @@ require (
github.com/projectdiscovery/wappalyzergo v0.1.14
github.com/redis/go-redis/v9 v9.1.0
github.com/seh-msft/burpxml v1.0.1
+ github.com/shurcooL/graphql v0.0.0-20230722043721-ed46e5a46466
github.com/stretchr/testify v1.9.0
github.com/tarunKoyalwar/goleak v0.0.0-20240429141123-0efa90dbdcf9
github.com/zmap/zgrab2 v0.1.8-0.20230806160807-97ba87c0e706
diff --git a/go.sum b/go.sum
index 77b6bdfd3b..70ee685efe 100644
--- a/go.sum
+++ b/go.sum
@@ -966,6 +966,8 @@ github.com/shoenig/go-m1cpu v0.1.6 h1:nxdKQNcEB6vzgA2E2bvzKIYRuNj7XNJ4S/aRSwKzFt
github.com/shoenig/go-m1cpu v0.1.6/go.mod h1:1JJMcUBvfNwpq05QDQVAnx3gUHr9IYF7GNg9SUEw2VQ=
github.com/shoenig/test v0.6.4 h1:kVTaSd7WLz5WZ2IaoM0RSzRsUD+m8wRR+5qvntpn4LU=
github.com/shoenig/test v0.6.4/go.mod h1:byHiCGXqrVaflBLAMq/srcZIHynQPQgeyvkvXnjqq0k=
+github.com/shurcooL/graphql v0.0.0-20230722043721-ed46e5a46466 h1:17JxqqJY66GmZVHkmAsGEkcIu0oCe3AM420QDgGwZx0=
+github.com/shurcooL/graphql v0.0.0-20230722043721-ed46e5a46466/go.mod h1:9dIRpgIY7hVhoqfe0/FcYp0bpInZaT7dc3BYOprrIUE=
github.com/shurcooL/sanitized_anchor_name v1.0.0/go.mod h1:1NzhyTcUVG4SuEtjjoZeVRXNmyL/1OwPU0+IJeTBvfc=
github.com/sirupsen/logrus v1.2.0/go.mod h1:LxeOpSwHxABJmUn/MG1IvRgCAasNZTLOkJPxbbu5VWo=
github.com/sirupsen/logrus v1.3.0/go.mod h1:LxeOpSwHxABJmUn/MG1IvRgCAasNZTLOkJPxbbu5VWo=
diff --git a/pkg/reporting/options.go b/pkg/reporting/options.go
index 06a749d658..c5090de014 100644
--- a/pkg/reporting/options.go
+++ b/pkg/reporting/options.go
@@ -12,6 +12,7 @@ import (
"github.com/projectdiscovery/nuclei/v3/pkg/reporting/trackers/github"
"github.com/projectdiscovery/nuclei/v3/pkg/reporting/trackers/gitlab"
"github.com/projectdiscovery/nuclei/v3/pkg/reporting/trackers/jira"
+ "github.com/projectdiscovery/nuclei/v3/pkg/reporting/trackers/linear"
"github.com/projectdiscovery/retryablehttp-go"
)
@@ -29,6 +30,8 @@ type Options struct {
Gitea *gitea.Options `yaml:"gitea"`
// Jira contains configuration options for Jira Issue Tracker
Jira *jira.Options `yaml:"jira"`
+ // Linear contains configuration options for Linear Issue Tracker
+ Linear *linear.Options `yaml:"linear"`
// MarkdownExporter contains configuration options for Markdown Exporter Module
MarkdownExporter *markdown.Options `yaml:"markdown"`
// SarifExporter contains configuration options for Sarif Exporter Module
diff --git a/pkg/reporting/reporting.go b/pkg/reporting/reporting.go
index 889f92f3f7..c6a7d63e10 100644
--- a/pkg/reporting/reporting.go
+++ b/pkg/reporting/reporting.go
@@ -28,6 +28,7 @@ import (
"github.com/projectdiscovery/nuclei/v3/pkg/reporting/trackers/github"
"github.com/projectdiscovery/nuclei/v3/pkg/reporting/trackers/gitlab"
"github.com/projectdiscovery/nuclei/v3/pkg/reporting/trackers/jira"
+ "github.com/projectdiscovery/nuclei/v3/pkg/reporting/trackers/linear"
errorutil "github.com/projectdiscovery/utils/errors"
fileutil "github.com/projectdiscovery/utils/file"
)
@@ -112,6 +113,15 @@ func New(options *Options, db string, doNotDedupe bool) (Client, error) {
}
client.trackers = append(client.trackers, tracker)
}
+ if options.Linear != nil {
+ options.Linear.HttpClient = options.HttpClient
+ options.Linear.OmitRaw = options.OmitRaw
+ tracker, err := linear.New(options.Linear)
+ if err != nil {
+ return nil, errorutil.NewWithErr(err).Wrap(ErrReportingClientCreation)
+ }
+ client.trackers = append(client.trackers, tracker)
+ }
if options.MarkdownExporter != nil {
exporter, err := markdown.New(options.MarkdownExporter)
if err != nil {
@@ -195,6 +205,7 @@ func CreateConfigIfNotExists() error {
GitLab: &gitlab.Options{},
Gitea: &gitea.Options{},
Jira: &jira.Options{},
+ Linear: &linear.Options{},
MarkdownExporter: &markdown.Options{},
SarifExporter: &sarif.Options{},
ElasticsearchExporter: &es.Options{},
diff --git a/pkg/reporting/trackers/linear/jsonutil/jsonutil.go b/pkg/reporting/trackers/linear/jsonutil/jsonutil.go
new file mode 100644
index 0000000000..cf66b7aa1e
--- /dev/null
+++ b/pkg/reporting/trackers/linear/jsonutil/jsonutil.go
@@ -0,0 +1,312 @@
+// Package jsonutil provides a function for decoding JSON
+// into a GraphQL query data structure.
+//
+// Taken from: https://github.com/shurcooL/graphql/blob/ed46e5a4646634fc16cb07c3b8db389542cc8847/internal/jsonutil/graphql.go
+package jsonutil
+
+import (
+ "bytes"
+ "encoding/json"
+ "errors"
+ "fmt"
+ "io"
+ "reflect"
+ "strings"
+)
+
+// UnmarshalGraphQL parses the JSON-encoded GraphQL response data and stores
+// the result in the GraphQL query data structure pointed to by v.
+//
+// The implementation is created on top of the JSON tokenizer available
+// in "encoding/json".Decoder.
+func UnmarshalGraphQL(data []byte, v any) error {
+ dec := json.NewDecoder(bytes.NewReader(data))
+ dec.UseNumber()
+ err := (&decoder{tokenizer: dec}).Decode(v)
+ if err != nil {
+ return err
+ }
+ tok, err := dec.Token()
+ switch err {
+ case io.EOF:
+ // Expect to get io.EOF. There shouldn't be any more
+ // tokens left after we've decoded v successfully.
+ return nil
+ case nil:
+ return fmt.Errorf("invalid token '%v' after top-level value", tok)
+ default:
+ return err
+ }
+}
+
+// decoder is a JSON decoder that performs custom unmarshaling behavior
+// for GraphQL query data structures. It's implemented on top of a JSON tokenizer.
+type decoder struct {
+ tokenizer interface {
+ Token() (json.Token, error)
+ }
+
+ // Stack of what part of input JSON we're in the middle of - objects, arrays.
+ parseState []json.Delim
+
+ // Stacks of values where to unmarshal.
+ // The top of each stack is the reflect.Value where to unmarshal next JSON value.
+ //
+ // The reason there's more than one stack is because we might be unmarshaling
+ // a single JSON value into multiple GraphQL fragments or embedded structs, so
+ // we keep track of them all.
+ vs [][]reflect.Value
+}
+
+// Decode decodes a single JSON value from d.tokenizer into v.
+func (d *decoder) Decode(v any) error {
+ rv := reflect.ValueOf(v)
+ if rv.Kind() != reflect.Ptr {
+ return fmt.Errorf("cannot decode into non-pointer %T", v)
+ }
+ d.vs = [][]reflect.Value{{rv.Elem()}}
+ return d.decode()
+}
+
+// decode decodes a single JSON value from d.tokenizer into d.vs.
+func (d *decoder) decode() error {
+ // The loop invariant is that the top of each d.vs stack
+ // is where we try to unmarshal the next JSON value we see.
+ for len(d.vs) > 0 {
+ tok, err := d.tokenizer.Token()
+ if err == io.EOF {
+ return errors.New("unexpected end of JSON input")
+ } else if err != nil {
+ return err
+ }
+
+ switch {
+
+ // Are we inside an object and seeing next key (rather than end of object)?
+ case d.state() == '{' && tok != json.Delim('}'):
+ key, ok := tok.(string)
+ if !ok {
+ return errors.New("unexpected non-key in JSON input")
+ }
+ someFieldExist := false
+ for i := range d.vs {
+ v := d.vs[i][len(d.vs[i])-1]
+ if v.Kind() == reflect.Ptr {
+ v = v.Elem()
+ }
+ var f reflect.Value
+ if v.Kind() == reflect.Struct {
+ f = fieldByGraphQLName(v, key)
+ if f.IsValid() {
+ someFieldExist = true
+ }
+ }
+ d.vs[i] = append(d.vs[i], f)
+ }
+ if !someFieldExist {
+ return fmt.Errorf("struct field for %q doesn't exist in any of %v places to unmarshal", key, len(d.vs))
+ }
+
+ // We've just consumed the current token, which was the key.
+ // Read the next token, which should be the value, and let the rest of code process it.
+ tok, err = d.tokenizer.Token()
+ if err == io.EOF {
+ return errors.New("unexpected end of JSON input")
+ } else if err != nil {
+ return err
+ }
+
+ // Are we inside an array and seeing next value (rather than end of array)?
+ case d.state() == '[' && tok != json.Delim(']'):
+ someSliceExist := false
+ for i := range d.vs {
+ v := d.vs[i][len(d.vs[i])-1]
+ if v.Kind() == reflect.Ptr {
+ v = v.Elem()
+ }
+ var f reflect.Value
+ if v.Kind() == reflect.Slice {
+ v.Set(reflect.Append(v, reflect.Zero(v.Type().Elem()))) // v = append(v, T).
+ f = v.Index(v.Len() - 1)
+ someSliceExist = true
+ }
+ d.vs[i] = append(d.vs[i], f)
+ }
+ if !someSliceExist {
+ return fmt.Errorf("slice doesn't exist in any of %v places to unmarshal", len(d.vs))
+ }
+ }
+
+ switch tok := tok.(type) {
+ case string, json.Number, bool, nil:
+ // Value.
+
+ for i := range d.vs {
+ v := d.vs[i][len(d.vs[i])-1]
+ if !v.IsValid() {
+ continue
+ }
+ err := unmarshalValue(tok, v)
+ if err != nil {
+ return err
+ }
+ }
+ d.popAllVs()
+
+ case json.Delim:
+ switch tok {
+ case '{':
+ // Start of object.
+
+ d.pushState(tok)
+
+ frontier := make([]reflect.Value, len(d.vs)) // Places to look for GraphQL fragments/embedded structs.
+ for i := range d.vs {
+ v := d.vs[i][len(d.vs[i])-1]
+ frontier[i] = v
+ // TODO: Do this recursively or not? Add a test case if needed.
+ if v.Kind() == reflect.Ptr && v.IsNil() {
+ v.Set(reflect.New(v.Type().Elem())) // v = new(T).
+ }
+ }
+ // Find GraphQL fragments/embedded structs recursively, adding to frontier
+ // as new ones are discovered and exploring them further.
+ for len(frontier) > 0 {
+ v := frontier[0]
+ frontier = frontier[1:]
+ if v.Kind() == reflect.Ptr {
+ v = v.Elem()
+ }
+ if v.Kind() != reflect.Struct {
+ continue
+ }
+ for i := 0; i < v.NumField(); i++ {
+ if isGraphQLFragment(v.Type().Field(i)) || v.Type().Field(i).Anonymous {
+ // Add GraphQL fragment or embedded struct.
+ d.vs = append(d.vs, []reflect.Value{v.Field(i)})
+ frontier = append(frontier, v.Field(i))
+ }
+ }
+ }
+ case '[':
+ // Start of array.
+
+ d.pushState(tok)
+
+ for i := range d.vs {
+ v := d.vs[i][len(d.vs[i])-1]
+ // TODO: Confirm this is needed, write a test case.
+ //if v.Kind() == reflect.Ptr && v.IsNil() {
+ // v.Set(reflect.New(v.Type().Elem())) // v = new(T).
+ //}
+
+ // Reset slice to empty (in case it had non-zero initial value).
+ if v.Kind() == reflect.Ptr {
+ v = v.Elem()
+ }
+ if v.Kind() != reflect.Slice {
+ continue
+ }
+ v.Set(reflect.MakeSlice(v.Type(), 0, 0)) // v = make(T, 0, 0).
+ }
+ case '}', ']':
+ // End of object or array.
+ d.popAllVs()
+ d.popState()
+ default:
+ return errors.New("unexpected delimiter in JSON input")
+ }
+ default:
+ return errors.New("unexpected token in JSON input")
+ }
+ }
+ return nil
+}
+
+// pushState pushes a new parse state s onto the stack.
+func (d *decoder) pushState(s json.Delim) {
+ d.parseState = append(d.parseState, s)
+}
+
+// popState pops a parse state (already obtained) off the stack.
+// The stack must be non-empty.
+func (d *decoder) popState() {
+ d.parseState = d.parseState[:len(d.parseState)-1]
+}
+
+// state reports the parse state on top of stack, or 0 if empty.
+func (d *decoder) state() json.Delim {
+ if len(d.parseState) == 0 {
+ return 0
+ }
+ return d.parseState[len(d.parseState)-1]
+}
+
+// popAllVs pops from all d.vs stacks, keeping only non-empty ones.
+func (d *decoder) popAllVs() {
+ var nonEmpty [][]reflect.Value
+ for i := range d.vs {
+ d.vs[i] = d.vs[i][:len(d.vs[i])-1]
+ if len(d.vs[i]) > 0 {
+ nonEmpty = append(nonEmpty, d.vs[i])
+ }
+ }
+ d.vs = nonEmpty
+}
+
+// fieldByGraphQLName returns an exported struct field of struct v
+// that matches GraphQL name, or invalid reflect.Value if none found.
+func fieldByGraphQLName(v reflect.Value, name string) reflect.Value {
+ for i := 0; i < v.NumField(); i++ {
+ if v.Type().Field(i).PkgPath != "" {
+ // Skip unexported field.
+ continue
+ }
+ if hasGraphQLName(v.Type().Field(i), name) {
+ return v.Field(i)
+ }
+ }
+ return reflect.Value{}
+}
+
+// hasGraphQLName reports whether struct field f has GraphQL name.
+func hasGraphQLName(f reflect.StructField, name string) bool {
+ value, ok := f.Tag.Lookup("graphql")
+ if !ok {
+ // TODO: caseconv package is relatively slow. Optimize it, then consider using it here.
+ //return caseconv.MixedCapsToLowerCamelCase(f.Name) == name
+ return strings.EqualFold(f.Name, name)
+ }
+ value = strings.TrimSpace(value) // TODO: Parse better.
+ if strings.HasPrefix(value, "...") {
+ // GraphQL fragment. It doesn't have a name.
+ return false
+ }
+ // Cut off anything that follows the field name,
+ // such as field arguments, aliases, directives.
+ if i := strings.IndexAny(value, "(:@"); i != -1 {
+ value = value[:i]
+ }
+ return strings.TrimSpace(value) == name
+}
+
+// isGraphQLFragment reports whether struct field f is a GraphQL fragment.
+func isGraphQLFragment(f reflect.StructField) bool {
+ value, ok := f.Tag.Lookup("graphql")
+ if !ok {
+ return false
+ }
+ value = strings.TrimSpace(value) // TODO: Parse better.
+ return strings.HasPrefix(value, "...")
+}
+
+// unmarshalValue unmarshals JSON value into v.
+// v must be addressable and not obtained by the use of unexported
+// struct fields, otherwise unmarshalValue will panic.
+func unmarshalValue(value json.Token, v reflect.Value) error {
+ b, err := json.Marshal(value) // TODO: Short-circuit (if profiling says it's worth it).
+ if err != nil {
+ return err
+ }
+ return json.Unmarshal(b, v.Addr().Interface())
+}
diff --git a/pkg/reporting/trackers/linear/linear.go b/pkg/reporting/trackers/linear/linear.go
new file mode 100644
index 0000000000..1ad9552f26
--- /dev/null
+++ b/pkg/reporting/trackers/linear/linear.go
@@ -0,0 +1,404 @@
+package linear
+
+import (
+ "bytes"
+ "context"
+ "encoding/json"
+ "errors"
+ "fmt"
+ "io"
+ "net/http"
+
+ "github.com/shurcooL/graphql"
+
+ "github.com/projectdiscovery/nuclei/v3/pkg/model/types/severity"
+ "github.com/projectdiscovery/nuclei/v3/pkg/output"
+ "github.com/projectdiscovery/nuclei/v3/pkg/reporting/exporters/markdown/util"
+ "github.com/projectdiscovery/nuclei/v3/pkg/reporting/format"
+ "github.com/projectdiscovery/nuclei/v3/pkg/reporting/trackers/filters"
+ "github.com/projectdiscovery/nuclei/v3/pkg/reporting/trackers/linear/jsonutil"
+ "github.com/projectdiscovery/nuclei/v3/pkg/types"
+ "github.com/projectdiscovery/retryablehttp-go"
+)
+
+// Integration is a client for linear issue tracker integration
+type Integration struct {
+ url string
+ httpclient *http.Client
+ options *Options
+}
+
+// Options contains the configuration options for linear issue tracker client
+type Options struct {
+ // APIKey is the API key for linear account.
+ APIKey string `yaml:"api-key" validate:"required"`
+
+ // AllowList contains a list of allowed events for this tracker
+ AllowList *filters.Filter `yaml:"allow-list"`
+ // DenyList contains a list of denied events for this tracker
+ DenyList *filters.Filter `yaml:"deny-list"`
+
+ // TeamID is the team id for the project
+ TeamID string `yaml:"team-id"`
+ // ProjectID is the project id for the project
+ ProjectID string `yaml:"project-id"`
+ // DuplicateIssueCheck is a bool to enable duplicate tracking issue check and update the newest
+ DuplicateIssueCheck bool `yaml:"duplicate-issue-check" default:"false"`
+
+ // OpenStateID is the id of the open state for the project
+ OpenStateID string `yaml:"open-state-id"`
+
+ HttpClient *retryablehttp.Client `yaml:"-"`
+ OmitRaw bool `yaml:"-"`
+}
+
+// New creates a new issue tracker integration client based on options.
+func New(options *Options) (*Integration, error) {
+ httpClient := &http.Client{
+ Transport: &addHeaderTransport{
+ T: http.DefaultTransport,
+ Key: options.APIKey,
+ },
+ }
+
+ integration := &Integration{
+ url: "https://api.linear.app/graphql",
+ options: options,
+ httpclient: httpClient,
+ }
+
+ return integration, nil
+}
+
+// CreateIssue creates an issue in the tracker
+func (i *Integration) CreateIssue(event *output.ResultEvent) (*filters.CreateIssueResponse, error) {
+ summary := format.Summary(event)
+ description := format.CreateReportDescription(event, util.MarkdownFormatter{}, i.options.OmitRaw)
+ _ = description
+
+ ctx := context.Background()
+
+ var err error
+ var existingIssue *linearIssue
+ if i.options.DuplicateIssueCheck {
+ existingIssue, err = i.findIssueByTitle(ctx, summary)
+ if err != nil && !errors.Is(err, io.EOF) {
+ return nil, err
+ }
+ }
+
+ if existingIssue == nil {
+ // Create a new issue
+ createdIssue, err := i.createIssueLinear(ctx, summary, description, priorityFromSeverity(event.Info.SeverityHolder.Severity))
+ if err != nil {
+ return nil, err
+ }
+ return &filters.CreateIssueResponse{
+ IssueID: types.ToString(createdIssue.ID),
+ IssueURL: types.ToString(createdIssue.URL),
+ }, nil
+ } else {
+ if existingIssue.State.Name == "Done" {
+ // Update the issue state to open
+ var issueUpdateInput struct {
+ StateID string `json:"stateId"`
+ }
+ issueUpdateInput.StateID = i.options.OpenStateID
+ variables := map[string]interface{}{
+ "issueUpdateInput": issueUpdateInput,
+ "issueID": types.ToString(existingIssue.ID),
+ }
+ var resp struct {
+ LastSyncID string `json:"lastSyncId"`
+ }
+ err := i.doGraphqlRequest(ctx, existingIssueUpdateStateMutation, &resp, variables, "IssueUpdate")
+ if err != nil {
+ return nil, fmt.Errorf("error reopening issue %s: %s", existingIssue.ID, err)
+ }
+ }
+
+ commentInput := map[string]interface{}{
+ "issueId": types.ToString(existingIssue.ID),
+ "body": description,
+ }
+ variables := map[string]interface{}{
+ "commentCreateInput": commentInput,
+ }
+ var resp struct {
+ LastSyncID string `json:"lastSyncId"`
+ }
+ err := i.doGraphqlRequest(ctx, commentCreateExistingTicketMutation, &resp, variables, "CommentCreate")
+ if err != nil {
+ return nil, fmt.Errorf("error commenting on issue %s: %s", existingIssue.ID, err)
+ }
+ return &filters.CreateIssueResponse{
+ IssueID: types.ToString(existingIssue.ID),
+ IssueURL: types.ToString(existingIssue.URL),
+ }, nil
+ }
+}
+
+func priorityFromSeverity(sev severity.Severity) float64 {
+ switch sev {
+ case severity.Critical:
+ return linearPriorityCritical
+ case severity.High:
+ return linearPriorityHigh
+ case severity.Medium:
+ return linearPriorityMedium
+ case severity.Low:
+ return linearPriorityLow
+ default:
+ return linearPriorityNone
+ }
+}
+
+type createIssueMutation struct {
+ IssueCreate struct {
+ Issue struct {
+ ID graphql.ID
+ Title graphql.String
+ Identifier graphql.String
+ State struct {
+ Name graphql.String
+ }
+ URL graphql.String
+ }
+ }
+}
+
+const (
+ createIssueGraphQLMutation = `mutation CreateIssue($input: IssueCreateInput!) {
+ issueCreate(input: $input) {
+ issue {
+ id
+ title
+ identifier
+ state {
+ name
+ }
+ url
+ }
+ }
+}`
+
+ searchExistingTicketQuery = `query ($teamID: ID, $projectID: ID, $title: String!) {
+ issues(filter: {
+ title: { eq: $title },
+ team: { id: { eq: $teamID } }
+ project: { id: { eq: $projectID } }
+ }) {
+ nodes {
+ id
+ title
+ identifier
+ state {
+ name
+ }
+ url
+ }
+ }
+}
+`
+
+ existingIssueUpdateStateMutation = `mutation IssueUpdate($issueUpdateInput: IssueUpdateInput!, $issueID: String!) {
+ issueUpdate(input: $issueUpdateInput, id: $issueID) {
+ lastSyncId
+ }
+}
+`
+
+ commentCreateExistingTicketMutation = `mutation CommentCreate($commentCreateInput: CommentCreateInput!) {
+ commentCreate(input: $commentCreateInput) {
+ lastSyncId
+ }
+}
+`
+)
+
+func (i *Integration) createIssueLinear(ctx context.Context, title, description string, priority float64) (*linearIssue, error) {
+ var mutation createIssueMutation
+ input := map[string]interface{}{
+ "title": title,
+ "description": description,
+ "priority": priority,
+ }
+ if i.options.TeamID != "" {
+ input["teamId"] = graphql.ID(i.options.TeamID)
+ }
+ if i.options.ProjectID != "" {
+ input["projectId"] = i.options.ProjectID
+ }
+
+ variables := map[string]interface{}{
+ "input": input,
+ }
+
+ err := i.doGraphqlRequest(ctx, createIssueGraphQLMutation, &mutation, variables, "CreateIssue")
+ if err != nil {
+ return nil, err
+ }
+
+ return &linearIssue{
+ ID: mutation.IssueCreate.Issue.ID,
+ Title: mutation.IssueCreate.Issue.Title,
+ Identifier: mutation.IssueCreate.Issue.Identifier,
+ State: struct {
+ Name graphql.String
+ }{
+ Name: mutation.IssueCreate.Issue.State.Name,
+ },
+ URL: mutation.IssueCreate.Issue.URL,
+ }, nil
+}
+
+func (i *Integration) findIssueByTitle(ctx context.Context, title string) (*linearIssue, error) {
+ var query findExistingIssuesSearch
+ variables := map[string]interface{}{
+ "title": graphql.String(title),
+ }
+ if i.options.TeamID != "" {
+ variables["teamId"] = graphql.ID(i.options.TeamID)
+ }
+ if i.options.ProjectID != "" {
+ variables["projectID"] = graphql.ID(i.options.ProjectID)
+ }
+
+ err := i.doGraphqlRequest(ctx, searchExistingTicketQuery, &query, variables, "")
+ if err != nil {
+ return nil, err
+ }
+
+ if len(query.Issues.Nodes) > 0 {
+ return &query.Issues.Nodes[0], nil
+ }
+ return nil, io.EOF
+}
+
+func (i *Integration) Name() string {
+ return "linear"
+}
+
+func (i *Integration) CloseIssue(event *output.ResultEvent) error {
+ // TODO: Unimplemented for now as not used in many places
+ // and overhead of maintaining our own API for this.
+ // This is too much code as it is :(
+ return nil
+}
+
+// ShouldFilter determines if an issue should be logged to this tracker
+func (i *Integration) ShouldFilter(event *output.ResultEvent) bool {
+ if i.options.AllowList != nil && !i.options.AllowList.GetMatch(event) {
+ return false
+ }
+
+ if i.options.DenyList != nil && i.options.DenyList.GetMatch(event) {
+ return false
+ }
+
+ return true
+}
+
+type linearIssue struct {
+ ID graphql.ID
+ Title graphql.String
+ Identifier graphql.String
+ State struct {
+ Name graphql.String
+ }
+ URL graphql.String
+}
+
+type findExistingIssuesSearch struct {
+ Issues struct {
+ Nodes []linearIssue
+ }
+}
+
+// Custom transport to add the API key to the header
+type addHeaderTransport struct {
+ T http.RoundTripper
+ Key string
+}
+
+func (adt *addHeaderTransport) RoundTrip(req *http.Request) (*http.Response, error) {
+ req.Header.Add("Authorization", adt.Key)
+ return adt.T.RoundTrip(req)
+}
+
+const (
+ linearPriorityNone = float64(0)
+ linearPriorityCritical = float64(1)
+ linearPriorityHigh = float64(2)
+ linearPriorityMedium = float64(3)
+ linearPriorityLow = float64(4)
+)
+
+// errors represents the "errors" array in a response from a GraphQL server.
+// If returned via error interface, the slice is expected to contain at least 1 element.
+//
+// Specification: https://spec.graphql.org/October2021/#sec-Errors.
+type errorsGraphql []struct {
+ Message string
+ Locations []struct {
+ Line int
+ Column int
+ }
+}
+
+// Error implements error interface.
+func (e errorsGraphql) Error() string {
+ return e[0].Message
+}
+
+// do executes a single GraphQL operation.
+func (i *Integration) doGraphqlRequest(ctx context.Context, query string, v any, variables map[string]any, operationName string) error {
+ in := struct {
+ Query string `json:"query"`
+ Variables map[string]any `json:"variables,omitempty"`
+ OperationName string `json:"operationName,omitempty"`
+ }{
+ Query: query,
+ Variables: variables,
+ OperationName: operationName,
+ }
+
+ var buf bytes.Buffer
+ err := json.NewEncoder(&buf).Encode(in)
+ if err != nil {
+ return err
+ }
+ req, err := http.NewRequestWithContext(ctx, http.MethodPost, i.url, &buf)
+ if err != nil {
+ return err
+ }
+ req.Header.Set("Content-Type", "application/json")
+ resp, err := i.httpclient.Do(req)
+ if err != nil {
+ return err
+ }
+ defer resp.Body.Close()
+ if resp.StatusCode != http.StatusOK {
+ body, _ := io.ReadAll(resp.Body)
+ return fmt.Errorf("non-200 OK status code: %v body: %q", resp.Status, body)
+ }
+ var out struct {
+ Data *json.RawMessage
+ Errors errorsGraphql
+ //Extensions any // Unused.
+ }
+ err = json.NewDecoder(resp.Body).Decode(&out)
+ if err != nil {
+ return err
+ }
+ if out.Data != nil {
+ err := jsonutil.UnmarshalGraphQL(*out.Data, v)
+ if err != nil {
+ return err
+ }
+ }
+ if len(out.Errors) > 0 {
+ return out.Errors
+ }
+ return nil
+}
From 39f8be2125c5f7e988f0a56884b908cee07f974a Mon Sep 17 00:00:00 2001
From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com>
Date: Thu, 12 Sep 2024 10:05:56 +0000
Subject: [PATCH 194/283] chore(deps): bump
github.com/projectdiscovery/wappalyzergo
Bumps [github.com/projectdiscovery/wappalyzergo](https://github.com/projectdiscovery/wappalyzergo) from 0.1.14 to 0.1.18.
- [Release notes](https://github.com/projectdiscovery/wappalyzergo/releases)
- [Commits](https://github.com/projectdiscovery/wappalyzergo/compare/v0.1.14...v0.1.18)
---
updated-dependencies:
- dependency-name: github.com/projectdiscovery/wappalyzergo
dependency-type: direct:production
update-type: version-update:semver-patch
...
Signed-off-by: dependabot[bot]
---
go.mod | 2 +-
go.sum | 6 ++----
2 files changed, 3 insertions(+), 5 deletions(-)
diff --git a/go.mod b/go.mod
index 7b5e247bf7..b0bbb7570b 100644
--- a/go.mod
+++ b/go.mod
@@ -98,7 +98,7 @@ require (
github.com/projectdiscovery/uncover v1.0.9
github.com/projectdiscovery/useragent v0.0.65
github.com/projectdiscovery/utils v0.2.8
- github.com/projectdiscovery/wappalyzergo v0.1.14
+ github.com/projectdiscovery/wappalyzergo v0.1.18
github.com/redis/go-redis/v9 v9.1.0
github.com/seh-msft/burpxml v1.0.1
github.com/shurcooL/graphql v0.0.0-20230722043721-ed46e5a46466
diff --git a/go.sum b/go.sum
index 70ee685efe..5f64afe1e8 100644
--- a/go.sum
+++ b/go.sum
@@ -625,8 +625,6 @@ github.com/kevinburke/ssh_config v1.2.0/go.mod h1:CT57kijsi8u/K/BOFA39wgDQJ9CxiF
github.com/kisielk/errcheck v1.1.0/go.mod h1:EZBBE59ingxPouuu3KfxchcWSUPOHkagtvWXihfKN4Q=
github.com/kisielk/errcheck v1.5.0/go.mod h1:pFxgyoBC7bSaBwPgfKdkLd5X25qrDl4LWUI2bnpBCr8=
github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck=
-github.com/kitabisa/go-ci v1.0.2 h1:rqHf8KEbQOxVb998TbqGRo70Z7ol44io7/jLYJUvKp8=
-github.com/kitabisa/go-ci v1.0.2/go.mod h1:e3wBSzaJbcifXrr/Gw2ZBLn44MmeqP5WySwXyHlCK/U=
github.com/kitabisa/go-ci v1.0.3 h1:JmIUIvcercRQc/9x/v02ydCCqU4MadSHaNaOF8T2pGA=
github.com/kitabisa/go-ci v1.0.3/go.mod h1:e3wBSzaJbcifXrr/Gw2ZBLn44MmeqP5WySwXyHlCK/U=
github.com/klauspost/compress v1.4.1/go.mod h1:RyIbtBH6LamlWaDj8nUwkbUhJ87Yi3uG0guNDohfE1A=
@@ -890,8 +888,8 @@ github.com/projectdiscovery/useragent v0.0.65 h1:x78ZwWdqpzokOHxLITUXvq+ljkTKc19
github.com/projectdiscovery/useragent v0.0.65/go.mod h1:deOP8YLJU6SCzM8k+K8PjkcOF4Ux0spqyO4ODZGIT4A=
github.com/projectdiscovery/utils v0.2.8 h1:++NcCJ+lXEfNBHKBs6q+cWa8JrVS8cYdGcW9jOgZebI=
github.com/projectdiscovery/utils v0.2.8/go.mod h1:UYJ8GKbZaezPFRT/cOk01LreQZ1QK0fko1EUnSUiSGU=
-github.com/projectdiscovery/wappalyzergo v0.1.14 h1:nt1IM4RUmqeymsXk4h6BsZbKDoS2hjFvPkT2GaI1rz4=
-github.com/projectdiscovery/wappalyzergo v0.1.14/go.mod h1:/hzgxkBFTMe2wDbA93nFfoMjULw7/vIZ9QPSAnCgUa8=
+github.com/projectdiscovery/wappalyzergo v0.1.18 h1:fFgETis0HcsNE7wREaUPYP45JqIyHgGorJaVp1RH7g4=
+github.com/projectdiscovery/wappalyzergo v0.1.18/go.mod h1:/hzgxkBFTMe2wDbA93nFfoMjULw7/vIZ9QPSAnCgUa8=
github.com/projectdiscovery/yamldoc-go v1.0.4 h1:eZoESapnMw6WAHiVgRwNqvbJEfNHEH148uthhFbG5jE=
github.com/projectdiscovery/yamldoc-go v1.0.4/go.mod h1:8PIPRcUD55UbtQdcfFR1hpIGRWG0P7alClXNGt1TBik=
github.com/prometheus/client_golang v0.9.1/go.mod h1:7SWBe2y4D6OKWSNQJUaRYU/AaXPKyh/dDVn+NZz0KFw=
From bd6330f72aa45d1fb8bacc62fe9a41e0aeb92bb5 Mon Sep 17 00:00:00 2001
From: Ramana Reddy <90540245+RamanaReddy0M@users.noreply.github.com>
Date: Thu, 12 Sep 2024 16:13:49 +0530
Subject: [PATCH 195/283] feat: upload existing scan results (#5603)
* feat: upload existing scan results
* fix lint test
* misc update
---------
Co-authored-by: sandeep <8293321+ehsandeep@users.noreply.github.com>
---
cmd/nuclei/main.go | 11 +++++++++-
internal/runner/runner.go | 46 +++++++++++++++++++++++++++++++++++++++
pkg/types/types.go | 2 ++
3 files changed, 58 insertions(+), 1 deletion(-)
diff --git a/cmd/nuclei/main.go b/cmd/nuclei/main.go
index de29d6581e..8fe5b29573 100644
--- a/cmd/nuclei/main.go
+++ b/cmd/nuclei/main.go
@@ -123,6 +123,13 @@ func main() {
runner.ParseOptions(options)
+ if options.ScanUploadFile != "" {
+ if err := runner.UploadResultsToCloud(options); err != nil {
+ gologger.Fatal().Msgf("could not upload scan results to cloud dashboard: %s\n", err)
+ }
+ return
+ }
+
nucleiRunner, err := runner.New(options)
if err != nil {
gologger.Fatal().Msgf("Could not create runner: %s\n", err)
@@ -420,9 +427,11 @@ on extensive configurability, massive extensibility and ease of use.`)
flagSet.CreateGroup("cloud", "Cloud",
flagSet.DynamicVar(&pdcpauth, "auth", "true", "configure projectdiscovery cloud (pdcp) api key"),
flagSet.StringVarP(&options.TeamID, "team-id", "tid", _pdcp.TeamIDEnv, "upload scan results to given team id (optional)"),
- flagSet.BoolVarP(&options.EnableCloudUpload, "cloud-upload", "cup", false, "upload scan results to pdcp dashboard"),
+ flagSet.BoolVarP(&options.EnableCloudUpload, "cloud-upload", "cup", false, "upload scan results to pdcp dashboard [DEPRECATED use -dashboard]"),
flagSet.StringVarP(&options.ScanID, "scan-id", "sid", "", "upload scan results to existing scan id (optional)"),
flagSet.StringVarP(&options.ScanName, "scan-name", "sname", "", "scan name to set (optional)"),
+ flagSet.BoolVarP(&options.EnableCloudUpload, "dashboard", "pd", false, "upload / view nuclei results in projectdiscovery cloud (pdcp) UI dashboard"),
+ flagSet.StringVarP(&options.ScanUploadFile, "dashboard-upload", "pdu", "", "upload / view nuclei results file (jsonl) in projectdiscovery cloud (pdcp) UI dashboard"),
)
flagSet.CreateGroup("Authentication", "Authentication",
diff --git a/internal/runner/runner.go b/internal/runner/runner.go
index bfb2bc64be..b36d8ed584 100644
--- a/internal/runner/runner.go
+++ b/internal/runner/runner.go
@@ -784,6 +784,52 @@ func (r *Runner) SaveResumeConfig(path string) error {
return os.WriteFile(path, data, permissionutil.ConfigFilePermission)
}
+// upload existing scan results to cloud with progress
+func UploadResultsToCloud(options *types.Options) error {
+ h := &pdcpauth.PDCPCredHandler{}
+ creds, err := h.GetCreds()
+ if err != nil {
+ return errors.Wrap(err, "could not get credentials for cloud upload")
+ }
+ ctx := context.TODO()
+ uploadWriter, err := pdcp.NewUploadWriter(ctx, creds)
+ if err != nil {
+ return errors.Wrap(err, "could not create upload writer")
+ }
+ if options.ScanID != "" {
+ _ = uploadWriter.SetScanID(options.ScanID)
+ }
+ if options.ScanName != "" {
+ uploadWriter.SetScanName(options.ScanName)
+ }
+ if options.TeamID != "" {
+ uploadWriter.SetTeamID(options.TeamID)
+ }
+
+ // Open file to count the number of results first
+ file, err := os.Open(options.ScanUploadFile)
+ if err != nil {
+ return errors.Wrap(err, "could not open scan upload file")
+ }
+ defer file.Close()
+
+ gologger.Info().Msgf("Uploading scan results to cloud dashboard from %s", options.ScanUploadFile)
+ dec := json.NewDecoder(file)
+ for dec.More() {
+ var r output.ResultEvent
+ err := dec.Decode(&r)
+ if err != nil {
+ gologger.Warning().Msgf("Could not decode jsonl: %s\n", err)
+ continue
+ }
+ if err = uploadWriter.Write(&r); err != nil {
+ gologger.Warning().Msgf("[%s] failed to upload: %s\n", r.TemplateID, err)
+ }
+ }
+ uploadWriter.Close()
+ return nil
+}
+
type WalkFunc func(reflect.Value, reflect.StructField)
// Walk traverses a struct and executes a callback function on each value in the struct.
diff --git a/pkg/types/types.go b/pkg/types/types.go
index cab1aacf54..51375e7342 100644
--- a/pkg/types/types.go
+++ b/pkg/types/types.go
@@ -384,6 +384,8 @@ type Options struct {
ScanID string
// ScanName is the name of the scan to be uploaded
ScanName string
+ // ScanUploadFile is the jsonl file to upload scan results to cloud
+ ScanUploadFile string
// TeamID is the team ID to use for cloud upload
TeamID string
// JsConcurrency is the number of concurrent js routines to run
From 72da91a3999838fd32d1bd1703f203be5e143b34 Mon Sep 17 00:00:00 2001
From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com>
Date: Thu, 12 Sep 2024 16:14:27 +0530
Subject: [PATCH 196/283] chore(deps): bump github.com/projectdiscovery/hmap
from 0.0.56 to 0.0.58 (#5619)
Bumps [github.com/projectdiscovery/hmap](https://github.com/projectdiscovery/hmap) from 0.0.56 to 0.0.58.
- [Release notes](https://github.com/projectdiscovery/hmap/releases)
- [Commits](https://github.com/projectdiscovery/hmap/compare/v0.0.56...v0.0.58)
---
updated-dependencies:
- dependency-name: github.com/projectdiscovery/hmap
dependency-type: direct:production
update-type: version-update:semver-patch
...
Signed-off-by: dependabot[bot]
Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com>
---
go.mod | 2 +-
go.sum | 4 ++--
2 files changed, 3 insertions(+), 3 deletions(-)
diff --git a/go.mod b/go.mod
index b0bbb7570b..fc2934614e 100644
--- a/go.mod
+++ b/go.mod
@@ -21,7 +21,7 @@ require (
github.com/pkg/errors v0.9.1
github.com/projectdiscovery/clistats v0.1.1
github.com/projectdiscovery/fastdialer v0.2.7
- github.com/projectdiscovery/hmap v0.0.57
+ github.com/projectdiscovery/hmap v0.0.58
github.com/projectdiscovery/interactsh v1.2.0
github.com/projectdiscovery/rawhttp v0.1.65
github.com/projectdiscovery/retryabledns v1.0.74
diff --git a/go.sum b/go.sum
index 5f64afe1e8..b1a6fc5f92 100644
--- a/go.sum
+++ b/go.sum
@@ -850,8 +850,8 @@ github.com/projectdiscovery/gostruct v0.0.2 h1:s8gP8ApugGM4go1pA+sVlPDXaWqNP5BBD
github.com/projectdiscovery/gostruct v0.0.2/go.mod h1:H86peL4HKwMXcQQtEa6lmC8FuD9XFt6gkNR0B/Mu5PE=
github.com/projectdiscovery/gozero v0.0.2 h1:8fJeaCjxL9tpm33uG/RsCQs6HGM/NE6eA3cjkilRQ+E=
github.com/projectdiscovery/gozero v0.0.2/go.mod h1:d8bZvDWW07LWNYWrwjZ4OO1I0cpkfqaysyDfSs9ibK8=
-github.com/projectdiscovery/hmap v0.0.57 h1:n45T9Me/fL0fqdVZ+Pi/eVizBguNRCleThqxgzNPLrA=
-github.com/projectdiscovery/hmap v0.0.57/go.mod h1:aa7egkpPIBukf5tis936KSddiXfEvWUKYLzrtV3CH8M=
+github.com/projectdiscovery/hmap v0.0.58 h1:SoXmnmYS2egPSRgFKgUhHozu8QvPIUKAuoDpuii9jkw=
+github.com/projectdiscovery/hmap v0.0.58/go.mod h1:nQTelqkgxU6vuuU+qmQloMrDBxYZMt2TTO0fV86yXN4=
github.com/projectdiscovery/httpx v1.6.8 h1:k0Y5g3ue/7QbDP0+LykIxp/VhPDLfau3UEUyuxtP7qE=
github.com/projectdiscovery/httpx v1.6.8/go.mod h1:7BIsDxyRwkBjthqFmEajXrA5f3yb4tlVfLmpNdf0ZXA=
github.com/projectdiscovery/interactsh v1.2.0 h1:Al6jHiR+Usl9egYJDLJaWNHOcH8Rugk8gWMasc8Cmw8=
From a45e4bbd19b45d3154c8c11ebaf645d3a34c0ebd Mon Sep 17 00:00:00 2001
From: Dogan Can Bakir <65292895+dogancanbakir@users.noreply.github.com>
Date: Thu, 12 Sep 2024 14:35:32 +0300
Subject: [PATCH 197/283] move code around (#5626)
---
lib/sdk_private.go | 16 ++++++++--------
1 file changed, 8 insertions(+), 8 deletions(-)
diff --git a/lib/sdk_private.go b/lib/sdk_private.go
index b65ecf2a2f..cacd9a1ca1 100644
--- a/lib/sdk_private.go
+++ b/lib/sdk_private.go
@@ -108,14 +108,6 @@ func (e *NucleiEngine) init(ctx context.Context) error {
return err
}
- if e.opts.ProxyInternal && types.ProxyURL != "" || types.ProxySocksURL != "" {
- httpclient, err := httpclientpool.Get(e.opts, &httpclientpool.Configuration{})
- if err != nil {
- return err
- }
- e.httpClient = httpclient
- }
-
e.parser = templates.NewParser()
if sharedInit == nil || protocolstate.ShouldInit() {
@@ -126,6 +118,14 @@ func (e *NucleiEngine) init(ctx context.Context) error {
_ = protocolinit.Init(e.opts)
})
+ if e.opts.ProxyInternal && types.ProxyURL != "" || types.ProxySocksURL != "" {
+ httpclient, err := httpclientpool.Get(e.opts, &httpclientpool.Configuration{})
+ if err != nil {
+ return err
+ }
+ e.httpClient = httpclient
+ }
+
e.applyRequiredDefaults(ctx)
var err error
From 2ac9aaf8717fddc4a94e766d0136619e4cf141e3 Mon Sep 17 00:00:00 2001
From: Ice3man
Date: Fri, 13 Sep 2024 23:45:27 +0530
Subject: [PATCH 198/283] bugfix: fixed misc issues with linear integration
(#5630)
---
pkg/reporting/trackers/linear/linear.go | 9 +++++++--
1 file changed, 7 insertions(+), 2 deletions(-)
diff --git a/pkg/reporting/trackers/linear/linear.go b/pkg/reporting/trackers/linear/linear.go
index 1ad9552f26..11712039f8 100644
--- a/pkg/reporting/trackers/linear/linear.go
+++ b/pkg/reporting/trackers/linear/linear.go
@@ -109,7 +109,9 @@ func (i *Integration) CreateIssue(event *output.ResultEvent) (*filters.CreateIss
"issueID": types.ToString(existingIssue.ID),
}
var resp struct {
- LastSyncID string `json:"lastSyncId"`
+ IssueUpdate struct {
+ LastSyncID int `json:"lastSyncId"`
+ }
}
err := i.doGraphqlRequest(ctx, existingIssueUpdateStateMutation, &resp, variables, "IssueUpdate")
if err != nil {
@@ -125,7 +127,9 @@ func (i *Integration) CreateIssue(event *output.ResultEvent) (*filters.CreateIss
"commentCreateInput": commentInput,
}
var resp struct {
- LastSyncID string `json:"lastSyncId"`
+ CommentCreate struct {
+ LastSyncID int `json:"lastSyncId"`
+ }
}
err := i.doGraphqlRequest(ctx, commentCreateExistingTicketMutation, &resp, variables, "CommentCreate")
if err != nil {
@@ -387,6 +391,7 @@ func (i *Integration) doGraphqlRequest(ctx context.Context, query string, v any,
Errors errorsGraphql
//Extensions any // Unused.
}
+
err = json.NewDecoder(resp.Body).Decode(&out)
if err != nil {
return err
From 87e99be4f6ba63db85de76c9e1d2389e653a7956 Mon Sep 17 00:00:00 2001
From: Tarun Koyalwar <45962551+tarunKoyalwar@users.noreply.github.com>
Date: Sat, 14 Sep 2024 00:06:08 +0530
Subject: [PATCH 199/283] scan error formatting (#5628)
---
pkg/scan/scan_context.go | 27 +++++++++---------------
pkg/tmplexec/exec.go | 43 +++++++++++++++++----------------------
pkg/tmplexec/interface.go | 39 +++++++++++++++++++++++++++++++++++
3 files changed, 68 insertions(+), 41 deletions(-)
diff --git a/pkg/scan/scan_context.go b/pkg/scan/scan_context.go
index 45456ddcac..51b98007a6 100644
--- a/pkg/scan/scan_context.go
+++ b/pkg/scan/scan_context.go
@@ -8,6 +8,7 @@ import (
"github.com/projectdiscovery/nuclei/v3/pkg/output"
"github.com/projectdiscovery/nuclei/v3/pkg/protocols/common/contextargs"
+ "github.com/projectdiscovery/utils/errkit"
)
type ScanContextOption func(*ScanContext)
@@ -30,7 +31,7 @@ type ScanContext struct {
OnWarning func(string)
// unexported state fields
- errors []error
+ error error
warnings []string
events []*output.InternalWrappedEvent
results []*output.ResultEvent
@@ -52,8 +53,8 @@ func (s *ScanContext) Context() context.Context {
return s.ctx
}
-func (s *ScanContext) GenerateErrorMessage() string {
- return joinErrors(s.errors)
+func (s *ScanContext) GenerateErrorMessage() error {
+ return s.error
}
// GenerateResult returns final results slice from all events
@@ -94,13 +95,16 @@ func (s *ScanContext) LogError(err error) {
if err == nil {
return
}
-
if s.OnError != nil {
s.OnError(err)
}
- s.errors = append(s.errors, err)
+ if s.error == nil {
+ s.error = err
+ } else {
+ s.error = errkit.Append(s.error, err)
+ }
- errorMessage := s.GenerateErrorMessage()
+ errorMessage := s.GenerateErrorMessage().Error()
for _, result := range s.results {
result.Error = errorMessage
@@ -129,14 +133,3 @@ func (s *ScanContext) LogWarning(format string, args ...any) {
}
}
}
-
-// joinErrors joins multiple errors and returns a single error string
-func joinErrors(errors []error) string {
- var errorMessages []string
- for _, e := range errors {
- if e != nil {
- errorMessages = append(errorMessages, e.Error())
- }
- }
- return strings.Join(errorMessages, "; ")
-}
diff --git a/pkg/tmplexec/exec.go b/pkg/tmplexec/exec.go
index 4ca9badf70..149deaa4d5 100644
--- a/pkg/tmplexec/exec.go
+++ b/pkg/tmplexec/exec.go
@@ -20,7 +20,6 @@ import (
"github.com/projectdiscovery/nuclei/v3/pkg/tmplexec/flow"
"github.com/projectdiscovery/nuclei/v3/pkg/tmplexec/generic"
"github.com/projectdiscovery/nuclei/v3/pkg/tmplexec/multiproto"
- "github.com/projectdiscovery/nuclei/v3/pkg/types/nucleierr"
"github.com/projectdiscovery/utils/errkit"
)
@@ -207,7 +206,7 @@ func (e *TemplateExecuter) Execute(ctx *scan.ScanContext) (bool, error) {
ctx.LogError(errx)
if lastMatcherEvent != nil {
- lastMatcherEvent.InternalEvent["error"] = tryParseCause(fmt.Errorf("%s", ctx.GenerateErrorMessage()))
+ lastMatcherEvent.InternalEvent["error"] = getErrorCause(ctx.GenerateErrorMessage())
writeFailureCallback(lastMatcherEvent, e.options.Options.MatcherStatus)
}
@@ -222,7 +221,7 @@ func (e *TemplateExecuter) Execute(ctx *scan.ScanContext) (bool, error) {
Info: e.options.TemplateInfo,
Type: e.getTemplateType(),
Host: ctx.Input.MetaInput.Input,
- Error: tryParseCause(fmt.Errorf("%s", ctx.GenerateErrorMessage())),
+ Error: getErrorCause(ctx.GenerateErrorMessage()),
},
},
OperatorsResult: &operators.Result{
@@ -235,31 +234,27 @@ func (e *TemplateExecuter) Execute(ctx *scan.ScanContext) (bool, error) {
return executed.Load() || matched.Load(), errx
}
-// tryParseCause tries to parse the cause of given error
+// getErrorCause tries to parse the cause of given error
// this is legacy support due to use of errorutil in existing libraries
// but this should not be required once all libraries are updated
-func tryParseCause(err error) string {
- errStr := ""
- errX := errkit.FromError(err)
- if errX != nil {
- var errCause error
-
- if len(errX.Errors()) > 1 {
- errCause = errX.Errors()[0]
- }
- if errCause == nil {
- errCause = errX
+func getErrorCause(err error) string {
+ if err == nil {
+ return ""
+ }
+ errx := errkit.FromError(err)
+ var cause error
+ for _, e := range errx.Errors() {
+ if e != nil && strings.Contains(e.Error(), "context deadline exceeded") {
+ continue
}
-
- msg := strings.Trim(errCause.Error(), "{} ")
- parts := strings.Split(msg, ":")
- errCause = errkit.New("%s", parts[len(parts)-1])
- errKind := errkit.GetErrorKind(err, nucleierr.ErrTemplateLogic).String()
- errStr = errCause.Error()
- errStr = strings.TrimSpace(strings.Replace(errStr, "errKind="+errKind, "", -1))
+ cause = e
+ break
}
-
- return errStr
+ if cause == nil {
+ cause = errkit.Append(errkit.New("could not get error cause"), errx)
+ }
+ // parseScanError prettifies the error message and removes everything except the cause
+ return parseScanError(cause.Error())
}
// ExecuteWithResults executes the protocol requests and returns results instead of writing them.
diff --git a/pkg/tmplexec/interface.go b/pkg/tmplexec/interface.go
index 67f9621116..36f139cc57 100644
--- a/pkg/tmplexec/interface.go
+++ b/pkg/tmplexec/interface.go
@@ -1,10 +1,15 @@
package tmplexec
import (
+ "errors"
+ "regexp"
+ "strings"
+
"github.com/projectdiscovery/nuclei/v3/pkg/scan"
"github.com/projectdiscovery/nuclei/v3/pkg/tmplexec/flow"
"github.com/projectdiscovery/nuclei/v3/pkg/tmplexec/generic"
"github.com/projectdiscovery/nuclei/v3/pkg/tmplexec/multiproto"
+ "github.com/projectdiscovery/utils/errkit"
)
var (
@@ -30,3 +35,37 @@ type TemplateEngine interface {
// Name returns name of template engine
Name() string
}
+
+var (
+ // A temporary fix to remove errKind from error message
+ // this is because errkit is not used everywhere yet
+ reNoKind = regexp.MustCompile(`([\[][^][]+[\]]|errKind=[^ ]+) `)
+)
+
+// parseScanError parses given scan error and only returning the cause
+// instead of inefficient one
+func parseScanError(msg string) string {
+ if msg == "" {
+ return ""
+ }
+ if strings.HasPrefix(msg, "ReadStatusLine:") {
+ // last index is actual error (from rawhttp)
+ parts := strings.Split(msg, ":")
+ msg = strings.TrimSpace(parts[len(parts)-1])
+ }
+ if strings.Contains(msg, "read ") {
+ // same here
+ parts := strings.Split(msg, ":")
+ msg = strings.TrimSpace(parts[len(parts)-1])
+ }
+ e := errkit.FromError(errors.New(msg))
+ for _, err := range e.Errors() {
+ if err != nil && strings.Contains(err.Error(), "context deadline exceeded") {
+ continue
+ }
+ msg = reNoKind.ReplaceAllString(err.Error(), "")
+ return msg
+ }
+ wrapped := errkit.Append(errkit.New("failed to get error cause"), e).Error()
+ return reNoKind.ReplaceAllString(wrapped, "")
+}
From c8cc419e8111e9f964944cd050ab0c1d40f6e9cf Mon Sep 17 00:00:00 2001
From: Dogan Can Bakir <65292895+dogancanbakir@users.noreply.github.com>
Date: Sun, 15 Sep 2024 16:50:59 +0300
Subject: [PATCH 200/283] remove duplicate directive (#5635)
---
Dockerfile | 1 -
1 file changed, 1 deletion(-)
diff --git a/Dockerfile b/Dockerfile
index 45f92d4485..9ac12d7d23 100644
--- a/Dockerfile
+++ b/Dockerfile
@@ -3,7 +3,6 @@ FROM golang:1.21-alpine AS build-env
RUN apk add build-base
WORKDIR /app
COPY . /app
-WORKDIR /app
RUN go mod download
RUN go build ./cmd/nuclei
From 0a536e6ce00b183b4169fbbdcf22a206bd8f6517 Mon Sep 17 00:00:00 2001
From: Ramana Reddy <90540245+RamanaReddy0M@users.noreply.github.com>
Date: Thu, 19 Sep 2024 18:52:49 +0530
Subject: [PATCH 201/283] fix matcher-status tests (#5647)
---
cmd/integration-test/matcher-status.go | 7 ++++---
1 file changed, 4 insertions(+), 3 deletions(-)
diff --git a/cmd/integration-test/matcher-status.go b/cmd/integration-test/matcher-status.go
index b88763720a..d89f7c46b0 100644
--- a/cmd/integration-test/matcher-status.go
+++ b/cmd/integration-test/matcher-status.go
@@ -3,6 +3,7 @@ package main
import (
"encoding/json"
"fmt"
+ "strings"
"github.com/projectdiscovery/nuclei/v3/pkg/output"
"github.com/projectdiscovery/nuclei/v3/pkg/testutils"
@@ -26,9 +27,9 @@ func (h *httpNoAccess) Execute(filePath string) error {
}
event := &output.ResultEvent{}
_ = json.Unmarshal([]byte(results[0]), event)
-
- if event.Error != "no address found for host" {
- return fmt.Errorf("unexpected result: expecting \"no address found for host\" error but got none")
+ expectedError := "no address found for host"
+ if !strings.Contains(event.Error, expectedError) {
+ return fmt.Errorf("unexpected result: expecting \"%s\" error but got \"%s\"", expectedError, event.Error)
}
return nil
}
From 3d2f31a56fc87ccae84afa21f4b4a537e1849e18 Mon Sep 17 00:00:00 2001
From: Ramana Reddy <90540245+RamanaReddy0M@users.noreply.github.com>
Date: Thu, 19 Sep 2024 18:58:20 +0530
Subject: [PATCH 202/283] fix missing template_url for pd signed templates when
executed from custom path (#5644)
---
pkg/catalog/loader/loader.go | 3 ++-
pkg/keys/key.go | 2 ++
pkg/output/output.go | 5 +++--
pkg/protocols/code/code.go | 1 +
pkg/protocols/dns/operators.go | 1 +
pkg/protocols/file/operators.go | 1 +
pkg/protocols/headless/operators.go | 1 +
pkg/protocols/http/operators.go | 1 +
pkg/protocols/javascript/js.go | 1 +
pkg/protocols/network/operators.go | 15 ++++++++-------
pkg/protocols/offlinehttp/operators.go | 1 +
pkg/protocols/protocols.go | 2 ++
pkg/protocols/ssl/ssl.go | 1 +
pkg/protocols/websocket/websocket.go | 1 +
pkg/protocols/whois/whois.go | 1 +
pkg/templates/compile.go | 5 ++---
pkg/templates/workflows.go | 3 ++-
pkg/testutils/testutils.go | 2 +-
pkg/utils/template_path.go | 16 +++++++---------
19 files changed, 39 insertions(+), 24 deletions(-)
diff --git a/pkg/catalog/loader/loader.go b/pkg/catalog/loader/loader.go
index e1d8371e35..731ebbd68e 100644
--- a/pkg/catalog/loader/loader.go
+++ b/pkg/catalog/loader/loader.go
@@ -15,6 +15,7 @@ import (
"github.com/projectdiscovery/nuclei/v3/pkg/catalog"
"github.com/projectdiscovery/nuclei/v3/pkg/catalog/config"
"github.com/projectdiscovery/nuclei/v3/pkg/catalog/loader/filter"
+ "github.com/projectdiscovery/nuclei/v3/pkg/keys"
"github.com/projectdiscovery/nuclei/v3/pkg/model/types/severity"
"github.com/projectdiscovery/nuclei/v3/pkg/protocols"
"github.com/projectdiscovery/nuclei/v3/pkg/templates"
@@ -434,7 +435,7 @@ func (store *Store) LoadTemplatesWithTags(templatesList, tags []string) []*templ
// increment signed/unsigned counters
if tmpl.Verified {
if tmpl.TemplateVerifier == "" {
- templates.SignatureStats[templates.PDVerifier].Add(1)
+ templates.SignatureStats[keys.PDVerifier].Add(1)
} else {
templates.SignatureStats[tmpl.TemplateVerifier].Add(1)
}
diff --git a/pkg/keys/key.go b/pkg/keys/key.go
index bb0c900626..eac43f27ae 100644
--- a/pkg/keys/key.go
+++ b/pkg/keys/key.go
@@ -3,5 +3,7 @@ package keys
import _ "embed"
+const PDVerifier = "projectdiscovery/nuclei-templates"
+
//go:embed nuclei.crt
var NucleiCert []byte // public key for verifying digital signature of templates
diff --git a/pkg/output/output.go b/pkg/output/output.go
index 4f02d71c38..07c6f4c71f 100644
--- a/pkg/output/output.go
+++ b/pkg/output/output.go
@@ -190,6 +190,7 @@ type ResultEvent struct {
FuzzingPosition string `json:"fuzzing_position,omitempty"`
FileToIndexPosition map[string]int `json:"-"`
+ TemplateVerifier string `json:"-"`
Error string `json:"error,omitempty"`
}
@@ -263,7 +264,7 @@ func NewStandardWriter(options *types.Options) (*StandardWriter, error) {
func (w *StandardWriter) Write(event *ResultEvent) error {
// Enrich the result event with extra metadata on the template-path and url.
if event.TemplatePath != "" {
- event.Template, event.TemplateURL = utils.TemplatePathURL(types.ToString(event.TemplatePath), types.ToString(event.TemplateID))
+ event.Template, event.TemplateURL = utils.TemplatePathURL(types.ToString(event.TemplatePath), types.ToString(event.TemplateID), event.TemplateVerifier)
}
if len(w.KeysToRedact) > 0 {
@@ -435,7 +436,7 @@ func (w *StandardWriter) WriteFailure(wrappedEvent *InternalWrappedEvent) error
// if no results were found, manually create a failure event
event := wrappedEvent.InternalEvent
- templatePath, templateURL := utils.TemplatePathURL(types.ToString(event["template-path"]), types.ToString(event["template-id"]))
+ templatePath, templateURL := utils.TemplatePathURL(types.ToString(event["template-path"]), types.ToString(event["template-id"]), types.ToString(event["template-verifier"]))
var templateInfo model.Info
if event["template-info"] != nil {
templateInfo = event["template-info"].(model.Info)
diff --git a/pkg/protocols/code/code.go b/pkg/protocols/code/code.go
index 7736113dca..87a0128645 100644
--- a/pkg/protocols/code/code.go
+++ b/pkg/protocols/code/code.go
@@ -348,6 +348,7 @@ func (request *Request) MakeResultEventItem(wrapped *output.InternalWrappedEvent
TemplateID: types.ToString(request.options.TemplateID),
TemplatePath: types.ToString(request.options.TemplatePath),
Info: request.options.TemplateInfo,
+ TemplateVerifier: request.options.TemplateVerifier,
Type: types.ToString(wrapped.InternalEvent["type"]),
Matched: types.ToString(wrapped.InternalEvent["input"]),
Host: fields.Host,
diff --git a/pkg/protocols/dns/operators.go b/pkg/protocols/dns/operators.go
index 7b8467646b..0f18315303 100644
--- a/pkg/protocols/dns/operators.go
+++ b/pkg/protocols/dns/operators.go
@@ -114,6 +114,7 @@ func (request *Request) MakeResultEventItem(wrapped *output.InternalWrappedEvent
TemplateID: types.ToString(wrapped.InternalEvent["template-id"]),
TemplatePath: types.ToString(wrapped.InternalEvent["template-path"]),
Info: wrapped.InternalEvent["template-info"].(model.Info),
+ TemplateVerifier: request.options.TemplateVerifier,
Type: types.ToString(wrapped.InternalEvent["type"]),
Host: types.ToString(wrapped.InternalEvent["host"]),
Matched: types.ToString(wrapped.InternalEvent["matched"]),
diff --git a/pkg/protocols/file/operators.go b/pkg/protocols/file/operators.go
index ff18af0979..5dab75bb91 100644
--- a/pkg/protocols/file/operators.go
+++ b/pkg/protocols/file/operators.go
@@ -104,6 +104,7 @@ func (request *Request) MakeResultEventItem(wrapped *output.InternalWrappedEvent
TemplateID: types.ToString(wrapped.InternalEvent["template-id"]),
TemplatePath: types.ToString(wrapped.InternalEvent["template-path"]),
Info: wrapped.InternalEvent["template-info"].(model.Info),
+ TemplateVerifier: request.options.TemplateVerifier,
Type: types.ToString(wrapped.InternalEvent["type"]),
Path: types.ToString(wrapped.InternalEvent["path"]),
Matched: types.ToString(wrapped.InternalEvent["matched"]),
diff --git a/pkg/protocols/headless/operators.go b/pkg/protocols/headless/operators.go
index d1b790f99c..3ad30e9c55 100644
--- a/pkg/protocols/headless/operators.go
+++ b/pkg/protocols/headless/operators.go
@@ -137,6 +137,7 @@ func (request *Request) MakeResultEventItem(wrapped *output.InternalWrappedEvent
TemplateID: types.ToString(wrapped.InternalEvent["template-id"]),
TemplatePath: types.ToString(wrapped.InternalEvent["template-path"]),
Info: wrapped.InternalEvent["template-info"].(model.Info),
+ TemplateVerifier: request.options.TemplateVerifier,
Type: types.ToString(wrapped.InternalEvent["type"]),
Host: fields.Host,
Path: fields.Path,
diff --git a/pkg/protocols/http/operators.go b/pkg/protocols/http/operators.go
index 3de3e63e34..d630bfd8b0 100644
--- a/pkg/protocols/http/operators.go
+++ b/pkg/protocols/http/operators.go
@@ -170,6 +170,7 @@ func (request *Request) MakeResultEventItem(wrapped *output.InternalWrappedEvent
TemplateID: types.ToString(wrapped.InternalEvent["template-id"]),
TemplatePath: types.ToString(wrapped.InternalEvent["template-path"]),
Info: wrapped.InternalEvent["template-info"].(model.Info),
+ TemplateVerifier: request.options.TemplateVerifier,
Type: types.ToString(wrapped.InternalEvent["type"]),
Host: fields.Host,
Port: fields.Port,
diff --git a/pkg/protocols/javascript/js.go b/pkg/protocols/javascript/js.go
index fbcd1a6ff9..0bd26b88a5 100644
--- a/pkg/protocols/javascript/js.go
+++ b/pkg/protocols/javascript/js.go
@@ -766,6 +766,7 @@ func (request *Request) MakeResultEventItem(wrapped *output.InternalWrappedEvent
TemplateID: types.ToString(wrapped.InternalEvent["template-id"]),
TemplatePath: types.ToString(wrapped.InternalEvent["template-path"]),
Info: wrapped.InternalEvent["template-info"].(model.Info),
+ TemplateVerifier: request.options.TemplateVerifier,
Type: types.ToString(wrapped.InternalEvent["type"]),
Host: fields.Host,
Port: fields.Port,
diff --git a/pkg/protocols/network/operators.go b/pkg/protocols/network/operators.go
index 4ff22130f4..2aa19e5b3b 100644
--- a/pkg/protocols/network/operators.go
+++ b/pkg/protocols/network/operators.go
@@ -100,13 +100,14 @@ func (request *Request) MakeResultEventItem(wrapped *output.InternalWrappedEvent
fields.Ip = types.ToString(wrapped.InternalEvent["ip"])
}
data := &output.ResultEvent{
- TemplateID: types.ToString(wrapped.InternalEvent["template-id"]),
- TemplatePath: types.ToString(wrapped.InternalEvent["template-path"]),
- Info: wrapped.InternalEvent["template-info"].(model.Info),
- Type: types.ToString(wrapped.InternalEvent["type"]),
- Host: fields.Host,
- Port: fields.Port,
- URL: fields.URL,
+ TemplateID: types.ToString(wrapped.InternalEvent["template-id"]),
+ TemplatePath: types.ToString(wrapped.InternalEvent["template-path"]),
+ Info: wrapped.InternalEvent["template-info"].(model.Info),
+ TemplateVerifier: request.options.TemplateVerifier,
+ Type: types.ToString(wrapped.InternalEvent["type"]),
+ Host: fields.Host,
+ Port: fields.Port,
+ URL: fields.URL,
Matched: types.ToString(wrapped.InternalEvent["matched"]),
ExtractedResults: wrapped.OperatorsResult.OutputExtracts,
Metadata: wrapped.OperatorsResult.PayloadValues,
diff --git a/pkg/protocols/offlinehttp/operators.go b/pkg/protocols/offlinehttp/operators.go
index 0d906ff514..f69abc4412 100644
--- a/pkg/protocols/offlinehttp/operators.go
+++ b/pkg/protocols/offlinehttp/operators.go
@@ -142,6 +142,7 @@ func (request *Request) MakeResultEventItem(wrapped *output.InternalWrappedEvent
TemplateID: types.ToString(wrapped.InternalEvent["template-id"]),
TemplatePath: types.ToString(wrapped.InternalEvent["template-path"]),
Info: wrapped.InternalEvent["template-info"].(model.Info),
+ TemplateVerifier: request.options.TemplateVerifier,
Type: types.ToString(wrapped.InternalEvent["type"]),
Path: types.ToString(wrapped.InternalEvent["path"]),
Matched: types.ToString(wrapped.InternalEvent["matched"]),
diff --git a/pkg/protocols/protocols.go b/pkg/protocols/protocols.go
index 8b8854cd8b..a9d50c1481 100644
--- a/pkg/protocols/protocols.go
+++ b/pkg/protocols/protocols.go
@@ -61,6 +61,8 @@ type ExecutorOptions struct {
TemplatePath string
// TemplateInfo contains information block of the template request
TemplateInfo model.Info
+ // TemplateVerifier is the verifier for the template
+ TemplateVerifier string
// RawTemplate is the raw template for the request
RawTemplate []byte
// Output is a writer interface for writing output events from executer.
diff --git a/pkg/protocols/ssl/ssl.go b/pkg/protocols/ssl/ssl.go
index 50da53111b..681043d3bf 100644
--- a/pkg/protocols/ssl/ssl.go
+++ b/pkg/protocols/ssl/ssl.go
@@ -398,6 +398,7 @@ func (request *Request) MakeResultEventItem(wrapped *output.InternalWrappedEvent
TemplateID: types.ToString(wrapped.InternalEvent["template-id"]),
TemplatePath: types.ToString(wrapped.InternalEvent["template-path"]),
Info: wrapped.InternalEvent["template-info"].(model.Info),
+ TemplateVerifier: request.options.TemplateVerifier,
Type: types.ToString(wrapped.InternalEvent["type"]),
Host: fields.Host,
Port: fields.Port,
diff --git a/pkg/protocols/websocket/websocket.go b/pkg/protocols/websocket/websocket.go
index a956de0e3f..aa099ef43a 100644
--- a/pkg/protocols/websocket/websocket.go
+++ b/pkg/protocols/websocket/websocket.go
@@ -402,6 +402,7 @@ func (request *Request) MakeResultEventItem(wrapped *output.InternalWrappedEvent
TemplateID: types.ToString(request.options.TemplateID),
TemplatePath: types.ToString(request.options.TemplatePath),
Info: request.options.TemplateInfo,
+ TemplateVerifier: request.options.TemplateVerifier,
Type: types.ToString(wrapped.InternalEvent["type"]),
Host: fields.Host,
Port: fields.Port,
diff --git a/pkg/protocols/whois/whois.go b/pkg/protocols/whois/whois.go
index ce0bafe56c..9963ec19f8 100644
--- a/pkg/protocols/whois/whois.go
+++ b/pkg/protocols/whois/whois.go
@@ -177,6 +177,7 @@ func (request *Request) MakeResultEventItem(wrapped *output.InternalWrappedEvent
TemplateID: types.ToString(request.options.TemplateID),
TemplatePath: types.ToString(request.options.TemplatePath),
Info: request.options.TemplateInfo,
+ TemplateVerifier: request.options.TemplateVerifier,
Type: types.ToString(wrapped.InternalEvent["type"]),
Host: types.ToString(wrapped.InternalEvent["host"]),
Metadata: wrapped.OperatorsResult.PayloadValues,
diff --git a/pkg/templates/compile.go b/pkg/templates/compile.go
index 54708e7ea1..01af3a999b 100644
--- a/pkg/templates/compile.go
+++ b/pkg/templates/compile.go
@@ -36,8 +36,7 @@ var (
)
const (
- Unsigned = "unsigned"
- PDVerifier = "projectdiscovery/nuclei-templates"
+ Unsigned = "unsigned"
)
func init() {
@@ -420,7 +419,7 @@ func parseTemplate(data []byte, options protocols.ExecutorOptions) (*Template, e
break
}
}
-
+ options.TemplateVerifier = template.TemplateVerifier
if !(template.Verified && verifier.Identifier() == "projectdiscovery/nuclei-templates") {
template.Options.RawTemplate = data
}
diff --git a/pkg/templates/workflows.go b/pkg/templates/workflows.go
index bafb7aaf31..50029af18a 100644
--- a/pkg/templates/workflows.go
+++ b/pkg/templates/workflows.go
@@ -4,6 +4,7 @@ import (
"github.com/pkg/errors"
"github.com/projectdiscovery/gologger"
+ "github.com/projectdiscovery/nuclei/v3/pkg/keys"
"github.com/projectdiscovery/nuclei/v3/pkg/model"
"github.com/projectdiscovery/nuclei/v3/pkg/protocols"
"github.com/projectdiscovery/nuclei/v3/pkg/utils/stats"
@@ -105,7 +106,7 @@ func parseWorkflowTemplate(workflow *workflows.WorkflowTemplate, preprocessor Pr
// increment signed/unsigned counters
if template.Verified {
if template.TemplateVerifier == "" {
- SignatureStats[PDVerifier].Add(1)
+ SignatureStats[keys.PDVerifier].Add(1)
} else {
SignatureStats[template.TemplateVerifier].Add(1)
}
diff --git a/pkg/testutils/testutils.go b/pkg/testutils/testutils.go
index 930787aab9..a1ed685c87 100644
--- a/pkg/testutils/testutils.go
+++ b/pkg/testutils/testutils.go
@@ -167,7 +167,7 @@ func (m *MockOutputWriter) WriteFailure(wrappedEvent *output.InternalWrappedEven
// create event
event := wrappedEvent.InternalEvent
- templatePath, templateURL := utils.TemplatePathURL(types.ToString(event["template-path"]), types.ToString(event["template-id"]))
+ templatePath, templateURL := utils.TemplatePathURL(types.ToString(event["template-path"]), types.ToString(event["template-id"]), types.ToString(event["template-verifier"]))
var templateInfo model.Info
if ti, ok := event["template-info"].(model.Info); ok {
templateInfo = ti
diff --git a/pkg/utils/template_path.go b/pkg/utils/template_path.go
index f6ed8ae92c..6570d90f20 100644
--- a/pkg/utils/template_path.go
+++ b/pkg/utils/template_path.go
@@ -4,6 +4,7 @@ import (
"strings"
"github.com/projectdiscovery/nuclei/v3/pkg/catalog/config"
+ "github.com/projectdiscovery/nuclei/v3/pkg/keys"
)
const (
@@ -12,16 +13,13 @@ const (
)
// TemplatePathURL returns the Path and URL for the provided template
-func TemplatePathURL(fullPath, templateId string) (string, string) {
- var templateDirectory string
+func TemplatePathURL(fullPath, templateId, templateVerifier string) (path string, url string) {
configData := config.DefaultConfig
if configData.TemplatesDirectory != "" && strings.HasPrefix(fullPath, configData.TemplatesDirectory) {
- templateDirectory = configData.TemplatesDirectory
- } else {
- return "", ""
+ path = strings.TrimPrefix(strings.TrimPrefix(fullPath, configData.TemplatesDirectory), "/")
}
-
- finalPath := strings.TrimPrefix(strings.TrimPrefix(fullPath, templateDirectory), "/")
- templateURL := TemplatesRepoURL + templateId
- return finalPath, templateURL
+ if templateVerifier == keys.PDVerifier {
+ url = TemplatesRepoURL + templateId
+ }
+ return
}
From 4cd065df5f78d871562ab96d55a7fdcc6a99eb75 Mon Sep 17 00:00:00 2001
From: Dwi Siswanto
Date: Thu, 19 Sep 2024 20:31:12 +0700
Subject: [PATCH 203/283] feat(headless): supporting standard lifecycle events
(#5632)
* refactor(headless): use `WaitStable` for `waitload` action
Signed-off-by: Dwi Siswanto
* feat(headless): add `getNavigationFunc`
Signed-off-by: Dwi Siswanto
* feat(headless): add `WaitDOM` action
Signed-off-by: Dwi Siswanto
* feat(headless): add `WaitFMP` action
Signed-off-by: Dwi Siswanto
* feat(headless): add `WaitFCP` action
Signed-off-by: Dwi Siswanto
* feat(headless): add `WaitIdle` action
Signed-off-by: Dwi Siswanto
* refactor(headless): `ActionWaitLoad` waits for `proto.PageLifecycleEventNameLoad`
also rename `Page.WaitLoad` to `Page.WaitStable` method.
Signed-off-by: Dwi Siswanto
* feat(headless): add `WaitStable` action
Signed-off-by: Dwi Siswanto
* refactor(headless): supporting `duration` arg for `WaitStable` action
Signed-off-by: Dwi Siswanto
* chore: ignore `*.png`
Signed-off-by: Dwi Siswanto
* test(headless): update `TestActionScreenshot*`
call `ActionWaitFMP` instead of `WaitLoad` before take screenshot
Signed-off-by: Dwi Siswanto
* feat(headless): chained with `Timeout` when `WaitStable`
Signed-off-by: Dwi Siswanto
---------
Signed-off-by: Dwi Siswanto
---
.gitignore | 5 +-
pkg/protocols/headless/engine/action_types.go | 27 ++++++-
pkg/protocols/headless/engine/page_actions.go | 71 ++++++++++++++++---
.../headless/engine/page_actions_test.go | 4 +-
4 files changed, 92 insertions(+), 15 deletions(-)
diff --git a/.gitignore b/.gitignore
index 5a50568910..146b0a892c 100644
--- a/.gitignore
+++ b/.gitignore
@@ -39,4 +39,7 @@ dist
pkg/protocols/common/helpers/deserialization/testdata/Deserialize.class
pkg/protocols/common/helpers/deserialization/testdata/ValueObject.class
pkg/protocols/common/helpers/deserialization/testdata/ValueObject2.ser
-vendor
\ No newline at end of file
+vendor
+
+# Headless `screenshot` action
+*.png
\ No newline at end of file
diff --git a/pkg/protocols/headless/engine/action_types.go b/pkg/protocols/headless/engine/action_types.go
index d61e377431..d66fe91130 100644
--- a/pkg/protocols/headless/engine/action_types.go
+++ b/pkg/protocols/headless/engine/action_types.go
@@ -45,9 +45,24 @@ const (
// ActionFilesInput performs an action on a file input.
// name:files
ActionFilesInput
- // ActionWaitLoad waits for the page to stop loading.
+ // ActionWaitDOM waits for the HTML document has been completely loaded & parsed.
+ // name:waitdom
+ ActionWaitDOM
+ // ActionWaitFCP waits for the first piece of content (text, image, etc.) is painted on the screen.
+ // name:waitfcp
+ ActionWaitFCP
+ // ActionWaitFMP waits for page has rendered enough meaningful content to be useful to the user.
+ // name:waitfmp
+ ActionWaitFMP
+ // ActionWaitIdle waits for the network is completely idle (no ongoing network requests).
+ // name:waitidle
+ ActionWaitIdle
+ // ActionWaitLoad waits for the page and all its resources (like stylesheets and images) have finished loading.
// name:waitload
ActionWaitLoad
+ // ActionWaitStable waits until the page is stable.
+ // name:waitstable
+ ActionWaitStable
// ActionGetResource performs a get resource action on an element
// name:getresource
ActionGetResource
@@ -102,7 +117,12 @@ var ActionStringToAction = map[string]ActionType{
"time": ActionTimeInput,
"select": ActionSelectInput,
"files": ActionFilesInput,
+ "waitdom": ActionWaitDOM,
+ "waitfcp": ActionWaitFCP,
+ "waitfmp": ActionWaitFMP,
+ "waitidle": ActionWaitIdle,
"waitload": ActionWaitLoad,
+ "waitstable": ActionWaitStable,
"getresource": ActionGetResource,
"extract": ActionExtract,
"setmethod": ActionSetMethod,
@@ -129,7 +149,12 @@ var ActionToActionString = map[ActionType]string{
ActionTimeInput: "time",
ActionSelectInput: "select",
ActionFilesInput: "files",
+ ActionWaitDOM: "waitdom",
+ ActionWaitFCP: "waitfcp",
+ ActionWaitFMP: "waitfmp",
+ ActionWaitIdle: "waitidle",
ActionWaitLoad: "waitload",
+ ActionWaitStable: "waitstable",
ActionGetResource: "getresource",
ActionExtract: "extract",
ActionSetMethod: "setmethod",
diff --git a/pkg/protocols/headless/engine/page_actions.go b/pkg/protocols/headless/engine/page_actions.go
index b34ed18b94..18ed515662 100644
--- a/pkg/protocols/headless/engine/page_actions.go
+++ b/pkg/protocols/headless/engine/page_actions.go
@@ -94,8 +94,28 @@ func (p *Page) ExecuteActions(input *contextargs.Context, actions []*Action, var
err = p.TimeInputElement(act, outData)
case ActionSelectInput:
err = p.SelectInputElement(act, outData)
+ case ActionWaitDOM:
+ event := proto.PageLifecycleEventNameDOMContentLoaded
+ err = p.WaitPageLifecycleEvent(act, outData, event)
+ case ActionWaitFCP:
+ event := proto.PageLifecycleEventNameFirstContentfulPaint
+ err = p.WaitPageLifecycleEvent(act, outData, event)
+ case ActionWaitFMP:
+ event := proto.PageLifecycleEventNameFirstMeaningfulPaint
+ err = p.WaitPageLifecycleEvent(act, outData, event)
+ case ActionWaitIdle:
+ event := proto.PageLifecycleEventNameNetworkIdle
+ err = p.WaitPageLifecycleEvent(act, outData, event)
case ActionWaitLoad:
- err = p.WaitLoad(act, outData)
+ event := proto.PageLifecycleEventNameLoad
+ err = p.WaitPageLifecycleEvent(act, outData, event)
+ case ActionWaitStable:
+ err = p.WaitStable(act, outData)
+ // NOTE(dwisiswant0): Mapping `ActionWaitLoad` to `Page.WaitStable`,
+ // just in case waiting for the `proto.PageLifecycleEventNameLoad` event
+ // doesn't meet expectations.
+ // case ActionWaitLoad, ActionWaitStable:
+ // err = p.WaitStable(act, outData)
case ActionGetResource:
err = p.GetResource(act, outData)
case ActionExtract:
@@ -204,6 +224,17 @@ func createBackOffSleeper(pollTimeout, timeout time.Duration) utils.Sleeper {
}
}
+func getNavigationFunc(p *Page, act *Action, event proto.PageLifecycleEventName) (func(), error) {
+ dur, err := getTimeout(p, act)
+ if err != nil {
+ return nil, errors.Wrap(err, "Wrong timeout given")
+ }
+
+ fn := p.page.Timeout(dur).WaitNavigation(event)
+
+ return fn, nil
+}
+
func getTimeout(p *Page, act *Action) (time.Duration, error) {
return geTimeParameter(p, act, "timeout", 3, time.Second)
}
@@ -518,20 +549,38 @@ func (p *Page) SelectInputElement(act *Action, out ActionData) error {
return nil
}
-// WaitLoad waits for the page to load
-func (p *Page) WaitLoad(act *Action, out ActionData) error {
- p.page.Timeout(2 * time.Second).WaitNavigation(proto.PageLifecycleEventNameFirstMeaningfulPaint)()
-
- // Wait for the window.onload event and also wait for the network requests
- // to become idle for a maximum duration of 3 seconds. If the requests
- // do not finish,
- if err := p.page.WaitLoad(); err != nil {
- return errors.Wrap(err, "could not wait load event")
+// WaitPageLifecycleEvent waits for specified page lifecycle event name
+func (p *Page) WaitPageLifecycleEvent(act *Action, out ActionData, event proto.PageLifecycleEventName) error {
+ fn, err := getNavigationFunc(p, act, event)
+ if err != nil {
+ return err
}
- _ = p.page.WaitIdle(1 * time.Second)
+
+ fn()
+
return nil
}
+// WaitStable waits until the page is stable
+func (p *Page) WaitStable(act *Action, out ActionData) error {
+ var dur time.Duration = time.Second // default stable page duration: 1s
+
+ timeout, err := getTimeout(p, act)
+ if err != nil {
+ return errors.Wrap(err, "Wrong timeout given")
+ }
+
+ argDur := act.Data["duration"]
+ if argDur != "" {
+ dur, err = time.ParseDuration(argDur)
+ if err != nil {
+ dur = time.Second
+ }
+ }
+
+ return p.page.Timeout(timeout).WaitStable(dur)
+}
+
// GetResource gets a resource from an element from page.
func (p *Page) GetResource(act *Action, out ActionData) error {
element, err := p.pageElementBy(act.Data)
diff --git a/pkg/protocols/headless/engine/page_actions_test.go b/pkg/protocols/headless/engine/page_actions_test.go
index 3620bb2bb4..bc85ea7417 100644
--- a/pkg/protocols/headless/engine/page_actions_test.go
+++ b/pkg/protocols/headless/engine/page_actions_test.go
@@ -201,7 +201,7 @@ func TestActionScreenshot(t *testing.T) {
filePath := filepath.Join(os.TempDir(), "test.png")
actions := []*Action{
{ActionType: ActionTypeHolder{ActionType: ActionNavigate}, Data: map[string]string{"url": "{{BaseURL}}"}},
- {ActionType: ActionTypeHolder{ActionType: ActionWaitLoad}},
+ {ActionType: ActionTypeHolder{ActionType: ActionWaitFMP}},
{ActionType: ActionTypeHolder{ActionType: ActionScreenshot}, Data: map[string]string{"to": filePath}},
}
@@ -229,7 +229,7 @@ func TestActionScreenshotToDir(t *testing.T) {
actions := []*Action{
{ActionType: ActionTypeHolder{ActionType: ActionNavigate}, Data: map[string]string{"url": "{{BaseURL}}"}},
- {ActionType: ActionTypeHolder{ActionType: ActionWaitLoad}},
+ {ActionType: ActionTypeHolder{ActionType: ActionWaitFMP}},
{ActionType: ActionTypeHolder{ActionType: ActionScreenshot}, Data: map[string]string{"to": filePath, "mkdir": "true"}},
}
From 6347efa5900c34f80384d93df43d11562f9b76a2 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Levente=20Kov=C3=A1ts?=
<22732484+tovask@users.noreply.github.com>
Date: Thu, 19 Sep 2024 15:32:03 +0200
Subject: [PATCH 204/283] Clone Input when the workflow execution forks (#5621)
* clone Input when the workflow forks, add integration test
* fix line endings
---
cmd/integration-test/workflow.go | 40 +++++++++++++++++++
.../multimatch-value-share-template.yaml | 23 +++++++++++
.../multimatch-value-share-workflow.yaml | 21 ++++++++++
pkg/core/workflow_execute.go | 3 +-
4 files changed, 86 insertions(+), 1 deletion(-)
create mode 100644 integration_tests/workflow/multimatch-value-share-template.yaml
create mode 100644 integration_tests/workflow/multimatch-value-share-workflow.yaml
diff --git a/cmd/integration-test/workflow.go b/cmd/integration-test/workflow.go
index e11575f90e..442e5169f6 100644
--- a/cmd/integration-test/workflow.go
+++ b/cmd/integration-test/workflow.go
@@ -13,6 +13,7 @@ import (
"github.com/projectdiscovery/nuclei/v3/pkg/templates"
"github.com/projectdiscovery/nuclei/v3/pkg/templates/signer"
"github.com/projectdiscovery/nuclei/v3/pkg/testutils"
+ sliceutil "github.com/projectdiscovery/utils/slice"
)
var workflowTestcases = []TestCaseInfo{
@@ -25,6 +26,7 @@ var workflowTestcases = []TestCaseInfo{
{Path: "workflow/dns-value-share-workflow.yaml", TestCase: &workflowDnsKeyValueShare{}},
{Path: "workflow/code-value-share-workflow.yaml", TestCase: &workflowCodeKeyValueShare{}, DisableOn: isCodeDisabled}, // isCodeDisabled declared in code.go
{Path: "workflow/multiprotocol-value-share-workflow.yaml", TestCase: &workflowMultiProtocolKeyValueShare{}},
+ {Path: "workflow/multimatch-value-share-workflow.yaml", TestCase: &workflowMultiMatchKeyValueShare{}},
{Path: "workflow/shared-cookie.yaml", TestCase: &workflowSharedCookies{}},
}
@@ -229,6 +231,44 @@ func (h *workflowMultiProtocolKeyValueShare) Execute(filePath string) error {
return expectResultsCount(results, 2)
}
+type workflowMultiMatchKeyValueShare struct{}
+
+// Execute executes a test case and returns an error if occurred
+func (h *workflowMultiMatchKeyValueShare) Execute(filePath string) error {
+ var receivedData []string
+ router := httprouter.New()
+ router.GET("/", func(w http.ResponseWriter, r *http.Request, _ httprouter.Params) {
+ fmt.Fprintf(w, "This is test matcher text")
+ })
+ router.GET("/path1", func(w http.ResponseWriter, r *http.Request, _ httprouter.Params) {
+ fmt.Fprintf(w, "href=\"test-value-%s\"", r.URL.Query().Get("v"))
+ })
+ router.GET("/path2", func(w http.ResponseWriter, r *http.Request, _ httprouter.Params) {
+ body, _ := io.ReadAll(r.Body)
+ receivedData = append(receivedData, string(body))
+ fmt.Fprintf(w, "test-value")
+ })
+ ts := httptest.NewServer(router)
+ defer ts.Close()
+
+ results, err := testutils.RunNucleiWorkflowAndGetResults(filePath, ts.URL, debug)
+ if err != nil {
+ return err
+ }
+
+ // Check if we received the data from both request to /path1 and it is not overwritten by the later one.
+ // They will appear in brackets because of another bug: https://github.com/orgs/projectdiscovery/discussions/3766
+ if !sliceutil.Contains(receivedData, "[test-value-1]") || !sliceutil.Contains(receivedData, "[test-value-2]") {
+ return fmt.Errorf(
+ "incorrect data: did not receive both extracted data from the first request!\nReceived Data:\n\t%s\nResults:\n\t%s",
+ strings.Join(receivedData, "\n\t"),
+ strings.Join(results, "\n\t"),
+ )
+ }
+ // The number of expected results is 3: the workflow's Matcher Name based condition check forwards both match, and the other branch with simple subtemplates goes with one
+ return expectResultsCount(results, 3)
+}
+
type workflowSharedCookies struct{}
// Execute executes a test case and returns an error if occurred
diff --git a/integration_tests/workflow/multimatch-value-share-template.yaml b/integration_tests/workflow/multimatch-value-share-template.yaml
new file mode 100644
index 0000000000..fd66975e26
--- /dev/null
+++ b/integration_tests/workflow/multimatch-value-share-template.yaml
@@ -0,0 +1,23 @@
+id: multimatch-value-share-template
+
+info:
+ name: MultiMatch Value Share Template
+ author: tovask
+ severity: info
+
+http:
+ - path:
+ - "{{BaseURL}}/path1?v=1"
+ - "{{BaseURL}}/path1?v=2"
+ matchers:
+ - type: word
+ name: test-matcher
+ words:
+ - "href"
+ extractors:
+ - type: regex
+ part: body
+ name: extracted
+ regex:
+ - 'href="(.*)"'
+ group: 1
diff --git a/integration_tests/workflow/multimatch-value-share-workflow.yaml b/integration_tests/workflow/multimatch-value-share-workflow.yaml
new file mode 100644
index 0000000000..f197f28864
--- /dev/null
+++ b/integration_tests/workflow/multimatch-value-share-workflow.yaml
@@ -0,0 +1,21 @@
+id: multimatch-value-share-workflow
+
+info:
+ name: MultiMatch Value Share Workflow
+ author: tovask
+ severity: info
+ description: Workflow to test value sharing when multiple matches occur in the extractor template
+
+workflows:
+ - template: workflow/multimatch-value-share-template.yaml
+ subtemplates:
+ - template: workflow/match-1.yaml
+ subtemplates:
+ - template: workflow/http-value-share-template-2.yaml
+ - template: workflow/multimatch-value-share-template.yaml
+ matchers:
+ - name: test-matcher
+ subtemplates:
+ - template: workflow/match-1.yaml
+ subtemplates:
+ - template: workflow/http-value-share-template-2.yaml
diff --git a/pkg/core/workflow_execute.go b/pkg/core/workflow_execute.go
index 028df4d01f..0b5d7e8722 100644
--- a/pkg/core/workflow_execute.go
+++ b/pkg/core/workflow_execute.go
@@ -139,7 +139,8 @@ func (e *Engine) runWorkflowStep(template *workflows.WorkflowTemplate, ctx *scan
defer swg.Done()
// create a new context with the same input but with unset callbacks
- subCtx := scan.NewScanContext(ctx.Context(), ctx.Input)
+ // clone the Input so that other parallel executions won't overwrite the shared variables when subsequent templates are running
+ subCtx := scan.NewScanContext(ctx.Context(), ctx.Input.Clone())
if err := e.runWorkflowStep(subtemplate, subCtx, results, swg, w); err != nil {
gologger.Warning().Msgf(workflowStepExecutionError, subtemplate.Template, err)
}
From 1610d96bc238b505dd3b7369b702bcfdc51ad172 Mon Sep 17 00:00:00 2001
From: ghost
Date: Thu, 19 Sep 2024 13:34:08 +0000
Subject: [PATCH 205/283] Auto Generate Syntax Docs + JSONSchema [Thu Sep 19
13:34:08 UTC 2024] :robot:
---
SYNTAX-REFERENCE.md | 10 ++++++++++
nuclei-jsonschema.json | 5 +++++
pkg/templates/templates_doc.go | 5 +++++
3 files changed, 20 insertions(+)
diff --git a/SYNTAX-REFERENCE.md b/SYNTAX-REFERENCE.md
index c757321494..9ec413d3ed 100755
--- a/SYNTAX-REFERENCE.md
+++ b/SYNTAX-REFERENCE.md
@@ -3664,8 +3664,18 @@ Enum Values:
- files
+ - waitdom
+
+ - waitfcp
+
+ - waitfmp
+
+ - waitidle
+
- waitload
+ - waitstable
+
- getresource
- extract
diff --git a/nuclei-jsonschema.json b/nuclei-jsonschema.json
index 993a744502..b4ed1d2982 100644
--- a/nuclei-jsonschema.json
+++ b/nuclei-jsonschema.json
@@ -247,7 +247,12 @@
"time",
"select",
"files",
+ "waitdom",
+ "waitfcp",
+ "waitfmp",
+ "waitidle",
"waitload",
+ "waitstable",
"getresource",
"extract",
"setmethod",
diff --git a/pkg/templates/templates_doc.go b/pkg/templates/templates_doc.go
index b484dad565..827e583d3b 100644
--- a/pkg/templates/templates_doc.go
+++ b/pkg/templates/templates_doc.go
@@ -1603,7 +1603,12 @@ func init() {
"time",
"select",
"files",
+ "waitdom",
+ "waitfcp",
+ "waitfmp",
+ "waitidle",
"waitload",
+ "waitstable",
"getresource",
"extract",
"setmethod",
From b69de1577718daf397e8de3fd3465ebe082ef5d7 Mon Sep 17 00:00:00 2001
From: Ramana Reddy <90540245+RamanaReddy0M@users.noreply.github.com>
Date: Thu, 19 Sep 2024 19:29:19 +0530
Subject: [PATCH 206/283] fix: Parse OpenApi http security schemes on empty
values (#5606)
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit
* fix: parse openapi http security schemes on empty values
* minor
---------
Co-authored-by: Doğan Can Bakır
---
pkg/input/formats/openapi/generator.go | 13 ++++++++-----
1 file changed, 8 insertions(+), 5 deletions(-)
diff --git a/pkg/input/formats/openapi/generator.go b/pkg/input/formats/openapi/generator.go
index 63fef7e290..e22c078769 100644
--- a/pkg/input/formats/openapi/generator.go
+++ b/pkg/input/formats/openapi/generator.go
@@ -27,7 +27,8 @@ import (
)
const (
- globalAuth = "globalAuth"
+ globalAuth = "globalAuth"
+ DEFAULT_HTTP_SCHEME_HEADER = "Authorization"
)
// GenerateRequestsFromSchema generates http requests from an OpenAPI 3.0 document object
@@ -426,18 +427,20 @@ func GenerateParameterFromSecurityScheme(scheme *openapi3.SecuritySchemeRef) (*o
if !generic.EqualsAny(scheme.Value.Scheme, "basic", "bearer") {
return nil, errorutil.NewWithTag("openapi", "unsupported security scheme (%s) found in openapi file", scheme.Value.Scheme)
}
- if scheme.Value.Name == "" {
- return nil, errorutil.NewWithTag("openapi", "security scheme (%s) name is empty", scheme.Value.Scheme)
+ // HTTP authentication schemes basic or bearer use the Authorization header
+ headerName := scheme.Value.Name
+ if headerName == "" {
+ headerName = DEFAULT_HTTP_SCHEME_HEADER
}
// create parameters using the scheme
switch scheme.Value.Scheme {
case "basic":
- h := openapi3.NewHeaderParameter(scheme.Value.Name)
+ h := openapi3.NewHeaderParameter(headerName)
h.Required = true
h.Description = globalAuth // differentiator for normal variables and global auth
return h, nil
case "bearer":
- h := openapi3.NewHeaderParameter(scheme.Value.Name)
+ h := openapi3.NewHeaderParameter(headerName)
h.Required = true
h.Description = globalAuth // differentiator for normal variables and global auth
return h, nil
From 3eee9678d06c075049a259149422024ad861434a Mon Sep 17 00:00:00 2001
From: Ramana Reddy <90540245+RamanaReddy0M@users.noreply.github.com>
Date: Fri, 20 Sep 2024 23:04:42 +0530
Subject: [PATCH 207/283] fix loading dynamic auth templates on fuzzing (#5646)
---
internal/runner/lazy.go | 1 +
pkg/catalog/loader/loader.go | 11 ++++++++++-
2 files changed, 11 insertions(+), 1 deletion(-)
diff --git a/internal/runner/lazy.go b/internal/runner/lazy.go
index eb41374513..900850b673 100644
--- a/internal/runner/lazy.go
+++ b/internal/runner/lazy.go
@@ -51,6 +51,7 @@ func GetAuthTmplStore(opts types.Options, catalog catalog.Catalog, execOpts prot
opts.ExcludeProtocols = nil
opts.IncludeConditions = nil
cfg := loader.NewConfig(&opts, catalog, execOpts)
+ cfg.StoreId = loader.AuthStoreId
store, err := loader.New(cfg)
if err != nil {
return nil, errorutil.NewWithErr(err).Msgf("failed to initialize dynamic auth templates store")
diff --git a/pkg/catalog/loader/loader.go b/pkg/catalog/loader/loader.go
index 731ebbd68e..48f14a4054 100644
--- a/pkg/catalog/loader/loader.go
+++ b/pkg/catalog/loader/loader.go
@@ -33,6 +33,7 @@ import (
const (
httpPrefix = "http://"
httpsPrefix = "https://"
+ AuthStoreId = "auth_store"
)
var (
@@ -41,6 +42,7 @@ var (
// Config contains the configuration options for the loader
type Config struct {
+ StoreId string // used to set store id (optional)
Templates []string
TemplateURLs []string
Workflows []string
@@ -67,6 +69,7 @@ type Config struct {
// Store is a storage for loaded nuclei templates
type Store struct {
+ id string // id of the store (optional)
tagFilter *templates.TagFilter
pathFilter *filter.PathFilter
config *Config
@@ -132,6 +135,7 @@ func New(cfg *Config) (*Store, error) {
// Create a tag filter based on provided configuration
store := &Store{
+ id: cfg.StoreId,
config: cfg,
tagFilter: tagFilter,
pathFilter: filter.NewPathFilter(&filter.PathFilterConfig{
@@ -230,6 +234,10 @@ func (store *Store) ReadTemplateFromURI(uri string, remote bool) ([]byte, error)
}
}
+func (store *Store) ID() string {
+ return store.id
+}
+
// Templates returns all the templates in the store
func (store *Store) Templates() []*templates.Template {
return store.templates
@@ -472,7 +480,8 @@ func (store *Store) LoadTemplatesWithTags(templatesList, tags []string) []*templ
return
}
// DAST only templates
- if store.config.ExecutorOptions.Options.DAST {
+ // Skip DAST filter when loading auth templates
+ if store.ID() != AuthStoreId && store.config.ExecutorOptions.Options.DAST {
// check if the template is a DAST template
if parsed.IsFuzzing() {
loadTemplate(parsed)
From 694835c459517250c661399e0320e74f1db3387e Mon Sep 17 00:00:00 2001
From: Douglas Danger Manley
<127235272+doug-threatmate@users.noreply.github.com>
Date: Fri, 20 Sep 2024 17:11:22 -0400
Subject: [PATCH 208/283] Add more support for `fs.FS` in template parsing
(#5421)
* misc update
* chore(deps): bump github.com/gin-gonic/gin from 1.9.0 to 1.9.1 (#4252)
Bumps [github.com/gin-gonic/gin](https://github.com/gin-gonic/gin) from 1.9.0 to 1.9.1.
- [Release notes](https://github.com/gin-gonic/gin/releases)
- [Changelog](https://github.com/gin-gonic/gin/blob/master/CHANGELOG.md)
- [Commits](https://github.com/gin-gonic/gin/compare/v1.9.0...v1.9.1)
---
updated-dependencies:
- dependency-name: github.com/gin-gonic/gin
dependency-type: indirect
...
Signed-off-by: dependabot[bot]
Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com>
* chore(deps): bump github.com/docker/docker (#4316)
Bumps [github.com/docker/docker](https://github.com/docker/docker) from 24.0.5+incompatible to 24.0.7+incompatible.
- [Release notes](https://github.com/docker/docker/releases)
- [Commits](https://github.com/docker/docker/compare/v24.0.5...v24.0.7)
---
updated-dependencies:
- dependency-name: github.com/docker/docker
dependency-type: indirect
...
Signed-off-by: dependabot[bot]
Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com>
* fix README_CN.md typos (#4369)
* version update
* Add more support for `fs.FS` in the disk catalog
This adds more support for `fs.FS` in the disk catalog. This
fixes some places where direct `os` file-related calls were being
made to use the catalog interface instead.
Note that the JavaScript compiler *still* does not work in any
context where the `pkg/js/libs/fs` package is used. In particular,
the `ReadFilesFromDir` function is hard-coded to use the `os`
package and not respect the catalog.
* Remove some testing artifacts
* Wrap up
* Unwind other changes
* Add a LoadHelperFileFunction to Options
* Use a direct func
* Tweak validation
* Use a function type
---------
Signed-off-by: dependabot[bot]
Co-authored-by: sandeep <8293321+ehsandeep@users.noreply.github.com>
Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com>
Co-authored-by: Xc1Ym
Co-authored-by: Sandeep Singh
---
pkg/catalog/disk/find.go | 13 +++++++-
pkg/catalog/disk/path.go | 34 +++++++++++++++------
pkg/protocols/common/generators/validate.go | 7 +++++
pkg/types/types.go | 21 +++++++++++--
4 files changed, 62 insertions(+), 13 deletions(-)
diff --git a/pkg/catalog/disk/find.go b/pkg/catalog/disk/find.go
index a5d46ff078..0e4021b87c 100644
--- a/pkg/catalog/disk/find.go
+++ b/pkg/catalog/disk/find.go
@@ -216,12 +216,17 @@ func (c *DiskCatalog) findGlobPathMatches(absPath string, processed map[string]s
// is a file, it returns true otherwise false with no errors.
func (c *DiskCatalog) findFileMatches(absPath string, processed map[string]struct{}) (match string, matched bool, err error) {
if c.templatesFS != nil {
- absPath = strings.TrimPrefix(absPath, "/")
+ absPath = strings.Trim(absPath, "/")
}
var info fs.File
if c.templatesFS == nil {
info, err = os.Open(absPath)
} else {
+ // If we were given no path, then it's not a file, it's the root, and we can quietly return.
+ if absPath == "" {
+ return "", false, nil
+ }
+
info, err = c.templatesFS.Open(absPath)
}
if err != nil {
@@ -263,6 +268,12 @@ func (c *DiskCatalog) findDirectoryMatches(absPath string, processed map[string]
},
)
} else {
+ // For the special case of the root directory, we need to pass "." to `fs.WalkDir`.
+ if absPath == "" {
+ absPath = "."
+ }
+ absPath = strings.TrimSuffix(absPath, "/")
+
err = fs.WalkDir(
c.templatesFS,
absPath,
diff --git a/pkg/catalog/disk/path.go b/pkg/catalog/disk/path.go
index a55faee8f6..5c4b6dbb95 100644
--- a/pkg/catalog/disk/path.go
+++ b/pkg/catalog/disk/path.go
@@ -2,6 +2,7 @@ package disk
import (
"fmt"
+ "io/fs"
"os"
"path/filepath"
"strings"
@@ -21,6 +22,11 @@ func (c *DiskCatalog) ResolvePath(templateName, second string) (string, error) {
if filepath.IsAbs(templateName) {
return templateName, nil
}
+ if c.templatesFS != nil {
+ if potentialPath, err := c.tryResolve(templateName); err != errNoValidCombination {
+ return potentialPath, nil
+ }
+ }
if second != "" {
secondBasePath := filepath.Join(filepath.Dir(second), templateName)
if potentialPath, err := c.tryResolve(secondBasePath); err != errNoValidCombination {
@@ -28,17 +34,19 @@ func (c *DiskCatalog) ResolvePath(templateName, second string) (string, error) {
}
}
- curDirectory, err := os.Getwd()
- if err != nil {
- return "", err
- }
+ if c.templatesFS == nil {
+ curDirectory, err := os.Getwd()
+ if err != nil {
+ return "", err
+ }
- templatePath := filepath.Join(curDirectory, templateName)
- if potentialPath, err := c.tryResolve(templatePath); err != errNoValidCombination {
- return potentialPath, nil
+ templatePath := filepath.Join(curDirectory, templateName)
+ if potentialPath, err := c.tryResolve(templatePath); err != errNoValidCombination {
+ return potentialPath, nil
+ }
}
- templatePath = filepath.Join(config.DefaultConfig.GetTemplateDir(), templateName)
+ templatePath := filepath.Join(config.DefaultConfig.GetTemplateDir(), templateName)
if potentialPath, err := c.tryResolve(templatePath); err != errNoValidCombination {
return potentialPath, nil
}
@@ -50,8 +58,14 @@ var errNoValidCombination = errors.New("no valid combination found")
// tryResolve attempts to load locate the target by iterating across all the folders tree
func (c *DiskCatalog) tryResolve(fullPath string) (string, error) {
- if fileutil.FileOrFolderExists(fullPath) {
- return fullPath, nil
+ if c.templatesFS == nil {
+ if fileutil.FileOrFolderExists(fullPath) {
+ return fullPath, nil
+ }
+ } else {
+ if _, err := fs.Stat(c.templatesFS, fullPath); err == nil {
+ return fullPath, nil
+ }
}
return "", errNoValidCombination
}
diff --git a/pkg/protocols/common/generators/validate.go b/pkg/protocols/common/generators/validate.go
index adb537d289..0aa0737149 100644
--- a/pkg/protocols/common/generators/validate.go
+++ b/pkg/protocols/common/generators/validate.go
@@ -22,6 +22,13 @@ func (g *PayloadGenerator) validate(payloads map[string]interface{}, templatePat
return errors.New("invalid number of lines in payload")
}
+ // For historical reasons, "validate" checks to see if the payload file exist.
+ // If we're using a custom helper function, then we need to skip any validation beyond just checking the string syntax.
+ // Actually attempting to load the file will determine whether or not it exists.
+ if g.options.LoadHelperFileFunction != nil {
+ return nil
+ }
+
// check if it's a file and try to load it
if fileutil.FileExists(payloadType) {
continue
diff --git a/pkg/types/types.go b/pkg/types/types.go
index 51375e7342..9cc88f49ff 100644
--- a/pkg/types/types.go
+++ b/pkg/types/types.go
@@ -23,6 +23,9 @@ var (
ErrNoMoreRequests = io.EOF
)
+// LoadHelperFileFunction can be used to load a helper file.
+type LoadHelperFileFunction func(helperFile, templatePath string, catalog catalog.Catalog) (io.ReadCloser, error)
+
// Options contains the configuration options for nuclei scanner.
type Options struct {
// Tags contains a list of tags to execute templates for. Multiple paths
@@ -408,6 +411,9 @@ type Options struct {
HttpApiEndpoint string
// ListTemplateProfiles lists all available template profiles
ListTemplateProfiles bool
+ // LoadHelperFileFunction is a function that will be used to execute LoadHelperFile.
+ // If none is provided, then the default implementation will be used.
+ LoadHelperFileFunction LoadHelperFileFunction
// timeouts contains various types of timeouts used in nuclei
// these timeouts are derived from dial-timeout (-timeout) with known multipliers
// This is internally managed and does not need to be set by user by explicitly setting
@@ -540,10 +546,21 @@ func (options *Options) ParseHeadlessOptionalArguments() map[string]string {
return optionalArguments
}
-// LoadHelperFile loads a helper file needed for the template
+// LoadHelperFile loads a helper file needed for the template.
+//
+// If LoadHelperFileFunction is set, then that function will be used.
+// Otherwise, the default implementation will be used, which respects the sandbox rules and only loads files from allowed directories.
+func (options *Options) LoadHelperFile(helperFile, templatePath string, catalog catalog.Catalog) (io.ReadCloser, error) {
+ if options.LoadHelperFileFunction != nil {
+ return options.LoadHelperFileFunction(helperFile, templatePath, catalog)
+ }
+ return options.defaultLoadHelperFile(helperFile, templatePath, catalog)
+}
+
+// defaultLoadHelperFile loads a helper file needed for the template
// this respects the sandbox rules and only loads files from
// allowed directories
-func (options *Options) LoadHelperFile(helperFile, templatePath string, catalog catalog.Catalog) (io.ReadCloser, error) {
+func (options *Options) defaultLoadHelperFile(helperFile, templatePath string, catalog catalog.Catalog) (io.ReadCloser, error) {
if !options.AllowLocalFileAccess {
// if global file access is disabled try loading with restrictions
absPath, err := options.GetValidAbsPath(helperFile, templatePath)
From b18f99e4621141ba2b61c0ec82eb6d9c78602571 Mon Sep 17 00:00:00 2001
From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com>
Date: Mon, 23 Sep 2024 05:16:47 +0000
Subject: [PATCH 209/283] chore(deps): bump github.com/projectdiscovery/rawhttp
Bumps [github.com/projectdiscovery/rawhttp](https://github.com/projectdiscovery/rawhttp) from 0.1.65 to 0.1.67.
- [Release notes](https://github.com/projectdiscovery/rawhttp/releases)
- [Commits](https://github.com/projectdiscovery/rawhttp/compare/v0.1.65...v0.1.67)
---
updated-dependencies:
- dependency-name: github.com/projectdiscovery/rawhttp
dependency-type: direct:production
update-type: version-update:semver-patch
...
Signed-off-by: dependabot[bot]
---
go.mod | 12 ++++++------
go.sum | 24 ++++++++++++------------
2 files changed, 18 insertions(+), 18 deletions(-)
diff --git a/go.mod b/go.mod
index fc2934614e..0cdfc66be3 100644
--- a/go.mod
+++ b/go.mod
@@ -20,12 +20,12 @@ require (
github.com/olekukonko/tablewriter v0.0.5
github.com/pkg/errors v0.9.1
github.com/projectdiscovery/clistats v0.1.1
- github.com/projectdiscovery/fastdialer v0.2.7
+ github.com/projectdiscovery/fastdialer v0.2.9
github.com/projectdiscovery/hmap v0.0.58
github.com/projectdiscovery/interactsh v1.2.0
- github.com/projectdiscovery/rawhttp v0.1.65
- github.com/projectdiscovery/retryabledns v1.0.74
- github.com/projectdiscovery/retryablehttp-go v1.0.76
+ github.com/projectdiscovery/rawhttp v0.1.67
+ github.com/projectdiscovery/retryabledns v1.0.76
+ github.com/projectdiscovery/retryablehttp-go v1.0.78
github.com/projectdiscovery/yamldoc-go v1.0.4
github.com/remeh/sizedwaitgroup v1.0.0
github.com/rs/xid v1.5.0
@@ -85,7 +85,7 @@ require (
github.com/projectdiscovery/fasttemplate v0.0.2
github.com/projectdiscovery/go-smb2 v0.0.0-20240129202741-052cc450c6cb
github.com/projectdiscovery/goflags v0.1.64
- github.com/projectdiscovery/gologger v1.1.23
+ github.com/projectdiscovery/gologger v1.1.24
github.com/projectdiscovery/gostruct v0.0.2
github.com/projectdiscovery/gozero v0.0.2
github.com/projectdiscovery/httpx v1.6.8
@@ -97,7 +97,7 @@ require (
github.com/projectdiscovery/tlsx v1.1.7
github.com/projectdiscovery/uncover v1.0.9
github.com/projectdiscovery/useragent v0.0.65
- github.com/projectdiscovery/utils v0.2.8
+ github.com/projectdiscovery/utils v0.2.10
github.com/projectdiscovery/wappalyzergo v0.1.18
github.com/redis/go-redis/v9 v9.1.0
github.com/seh-msft/burpxml v1.0.1
diff --git a/go.sum b/go.sum
index b1a6fc5f92..b91861860e 100644
--- a/go.sum
+++ b/go.sum
@@ -834,8 +834,8 @@ github.com/projectdiscovery/clistats v0.1.1 h1:8mwbdbwTU4aT88TJvwIzTpiNeow3XnAB7
github.com/projectdiscovery/clistats v0.1.1/go.mod h1:4LtTC9Oy//RiuT1+76MfTg8Hqs7FQp1JIGBM3nHK6a0=
github.com/projectdiscovery/dsl v0.2.1 h1:TK3KD4jsg4YbvY7WJqnz1QyH4AOvAwezeBFOX97Evgk=
github.com/projectdiscovery/dsl v0.2.1/go.mod h1:IRQXsmi5/g1dDZ79//A9t2vrRtxm4frRSd5t8CZVSbI=
-github.com/projectdiscovery/fastdialer v0.2.7 h1:x7bEf/kDyVOuG9kHfJLMhWO3h7J+6fMDgLWkGYL8EFg=
-github.com/projectdiscovery/fastdialer v0.2.7/go.mod h1:7/BGdaw7sCekuBh6lLZqPuoO5N0Qaizwvex/PjO+V8o=
+github.com/projectdiscovery/fastdialer v0.2.9 h1:vDCqxVMCyUu3oVEizEK1K8K+CCcLkVDW3X2HfiWaVFA=
+github.com/projectdiscovery/fastdialer v0.2.9/go.mod h1:mYv5QaNBDDSHlZO9DI0niRMw+G5hUzwIhs8QixSElUI=
github.com/projectdiscovery/fasttemplate v0.0.2 h1:h2cISk5xDhlJEinlBQS6RRx0vOlOirB2y3Yu4PJzpiA=
github.com/projectdiscovery/fasttemplate v0.0.2/go.mod h1:XYWWVMxnItd+r0GbjA1GCsUopMw1/XusuQxdyAIHMCw=
github.com/projectdiscovery/freeport v0.0.6 h1:ROqzuXN8JPqkGdBueb3ah691nS2g2p7r3/3x2E33GbI=
@@ -844,8 +844,8 @@ github.com/projectdiscovery/go-smb2 v0.0.0-20240129202741-052cc450c6cb h1:rutG90
github.com/projectdiscovery/go-smb2 v0.0.0-20240129202741-052cc450c6cb/go.mod h1:FLjF1DmZ+POoGEiIQdWuYVwS++C/GwpX8YaCsTSm1RY=
github.com/projectdiscovery/goflags v0.1.64 h1:FDfwdt9N97Hi8OuhbkDlKtVttpc/CRMIWQVa08VsHsI=
github.com/projectdiscovery/goflags v0.1.64/go.mod h1:3FyHIVQtnycNOc1LE3O1jj/XR5XuMdF9QfHd0ujhnX4=
-github.com/projectdiscovery/gologger v1.1.23 h1:u9ksRgkWuUoaOqkk0Z1FngWriXH0Bnt73yYwFGmZo8E=
-github.com/projectdiscovery/gologger v1.1.23/go.mod h1:zcFt3u4U0kmhXou7cTXWwwu+yqVPX3GSABD1hqU0Ur8=
+github.com/projectdiscovery/gologger v1.1.24 h1:TmA4k9sA6ZvfyRnprZKQ0Uq34g//u5R9yTDPL9IzTOQ=
+github.com/projectdiscovery/gologger v1.1.24/go.mod h1:JA0JMJ+ply+J2wD062TN4h85thm6/28jAlrntwccKVU=
github.com/projectdiscovery/gostruct v0.0.2 h1:s8gP8ApugGM4go1pA+sVlPDXaWqNP5BBDDSv7VEdG1M=
github.com/projectdiscovery/gostruct v0.0.2/go.mod h1:H86peL4HKwMXcQQtEa6lmC8FuD9XFt6gkNR0B/Mu5PE=
github.com/projectdiscovery/gozero v0.0.2 h1:8fJeaCjxL9tpm33uG/RsCQs6HGM/NE6eA3cjkilRQ+E=
@@ -868,14 +868,14 @@ github.com/projectdiscovery/networkpolicy v0.0.9 h1:IrlDoYZagNNO8y+7iZeHT8k5izE+
github.com/projectdiscovery/networkpolicy v0.0.9/go.mod h1:XFJ2Lnv8BE/ziQCFjBHMsH1w6VmkPiQtk+NlBpdMU7M=
github.com/projectdiscovery/ratelimit v0.0.53 h1:1ci2zaVCtJ9daDi9IXjNwpDxl2InYq375role/xqUR4=
github.com/projectdiscovery/ratelimit v0.0.53/go.mod h1:4kNYTL7xKLo1qMYHTREUCBqHyvykFF+XBWutTh9fkwg=
-github.com/projectdiscovery/rawhttp v0.1.65 h1:/SGp+UbFBSDquEHgWGA8VRH3kYiM9uMFQjfLfXBKOfY=
-github.com/projectdiscovery/rawhttp v0.1.65/go.mod h1:X7kvZ1uTzdSHc5bb68qmmpit6DGuqc1wXQLfFNfRJq0=
+github.com/projectdiscovery/rawhttp v0.1.67 h1:HYzathMk3c8Y83hYjHM4GCBFbz/G+vABe0Lz6ajaowY=
+github.com/projectdiscovery/rawhttp v0.1.67/go.mod h1:5viJ6odzc9ZuEFppj/E7HdX+u99FoYlvXnhHyTNc7N0=
github.com/projectdiscovery/rdap v0.9.1-0.20221108103045-9865884d1917 h1:m03X4gBVSorSzvmm0bFa7gDV4QNSOWPL/fgZ4kTXBxk=
github.com/projectdiscovery/rdap v0.9.1-0.20221108103045-9865884d1917/go.mod h1:JxXtZC9e195awe7EynrcnBJmFoad/BNDzW9mzFkK8Sg=
-github.com/projectdiscovery/retryabledns v1.0.74 h1:4NtSjnVdPWT1NuabOMf5vjUoky3X1DwJN6iFanrtQnk=
-github.com/projectdiscovery/retryabledns v1.0.74/go.mod h1:GRBpKmWIds0UVeDpqSek2Q8RITuuIZWD5wHL4cN5geg=
-github.com/projectdiscovery/retryablehttp-go v1.0.76 h1:2+4QpWnXUZFBkoeCEBXtUjSeBTdgKtaGwP8LRtKAC3g=
-github.com/projectdiscovery/retryablehttp-go v1.0.76/go.mod h1:0qIv7Ld8WWSKWYXMsNAWRYU5gsxnxpBza3SjDgt6nVc=
+github.com/projectdiscovery/retryabledns v1.0.76 h1:6PyRCSW5SMUfE7uozZWckWuIDnBgLU/b1ofeWqeKONs=
+github.com/projectdiscovery/retryabledns v1.0.76/go.mod h1:vc4L2dng+nPLpQu7s0eqFc0OvyrZibuMNWFMA9bnN3w=
+github.com/projectdiscovery/retryablehttp-go v1.0.78 h1:If7/XjCWk893YrnTMaW69TNMsfE1Er3i1SWOkWbEk4E=
+github.com/projectdiscovery/retryablehttp-go v1.0.78/go.mod h1:NqzTdnGihSRkF9c/aXo+3qTJXEeNwnOuza6GrlPo9qw=
github.com/projectdiscovery/sarif v0.0.1 h1:C2Tyj0SGOKbCLgHrx83vaE6YkzXEVrMXYRGLkKCr/us=
github.com/projectdiscovery/sarif v0.0.1/go.mod h1:cEYlDu8amcPf6b9dSakcz2nNnJsoz4aR6peERwV+wuQ=
github.com/projectdiscovery/stringsutil v0.0.2 h1:uzmw3IVLJSMW1kEg8eCStG/cGbYYZAja8BH3LqqJXMA=
@@ -886,8 +886,8 @@ github.com/projectdiscovery/uncover v1.0.9 h1:s5RbkD/V4r8QcPkys4gTTqMuRSgXq0Jpre
github.com/projectdiscovery/uncover v1.0.9/go.mod h1:2PUF3SpB5QNIJ8epaB2xbRzkPaxEAWRDm3Ir2ijt81U=
github.com/projectdiscovery/useragent v0.0.65 h1:x78ZwWdqpzokOHxLITUXvq+ljkTKc19z3ILGtoV1N70=
github.com/projectdiscovery/useragent v0.0.65/go.mod h1:deOP8YLJU6SCzM8k+K8PjkcOF4Ux0spqyO4ODZGIT4A=
-github.com/projectdiscovery/utils v0.2.8 h1:++NcCJ+lXEfNBHKBs6q+cWa8JrVS8cYdGcW9jOgZebI=
-github.com/projectdiscovery/utils v0.2.8/go.mod h1:UYJ8GKbZaezPFRT/cOk01LreQZ1QK0fko1EUnSUiSGU=
+github.com/projectdiscovery/utils v0.2.10 h1:MFx9pAaSyYGjXNi46mdAD7ceSZ80E0AGeuBj8fXaAF4=
+github.com/projectdiscovery/utils v0.2.10/go.mod h1:jvTckFUQxZSiJrhACUuU63tqtd0tdBu3ng3QUVm/Ce0=
github.com/projectdiscovery/wappalyzergo v0.1.18 h1:fFgETis0HcsNE7wREaUPYP45JqIyHgGorJaVp1RH7g4=
github.com/projectdiscovery/wappalyzergo v0.1.18/go.mod h1:/hzgxkBFTMe2wDbA93nFfoMjULw7/vIZ9QPSAnCgUa8=
github.com/projectdiscovery/yamldoc-go v1.0.4 h1:eZoESapnMw6WAHiVgRwNqvbJEfNHEH148uthhFbG5jE=
From 182e000f2c1e9ba85d3f0b7b8b5861e9da8b5152 Mon Sep 17 00:00:00 2001
From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com>
Date: Mon, 23 Sep 2024 05:43:13 +0000
Subject: [PATCH 210/283] chore(deps): bump
github.com/projectdiscovery/ratelimit
Bumps [github.com/projectdiscovery/ratelimit](https://github.com/projectdiscovery/ratelimit) from 0.0.53 to 0.0.56.
- [Release notes](https://github.com/projectdiscovery/ratelimit/releases)
- [Commits](https://github.com/projectdiscovery/ratelimit/compare/v0.0.53...v0.0.56)
---
updated-dependencies:
- dependency-name: github.com/projectdiscovery/ratelimit
dependency-type: direct:production
update-type: version-update:semver-patch
...
Signed-off-by: dependabot[bot]
---
go.mod | 2 +-
go.sum | 4 ++--
2 files changed, 3 insertions(+), 3 deletions(-)
diff --git a/go.mod b/go.mod
index 0cdfc66be3..4c3f14ab40 100644
--- a/go.mod
+++ b/go.mod
@@ -91,7 +91,7 @@ require (
github.com/projectdiscovery/httpx v1.6.8
github.com/projectdiscovery/mapcidr v1.1.34
github.com/projectdiscovery/n3iwf v0.0.0-20230523120440-b8cd232ff1f5
- github.com/projectdiscovery/ratelimit v0.0.53
+ github.com/projectdiscovery/ratelimit v0.0.56
github.com/projectdiscovery/rdap v0.9.1-0.20221108103045-9865884d1917
github.com/projectdiscovery/sarif v0.0.1
github.com/projectdiscovery/tlsx v1.1.7
diff --git a/go.sum b/go.sum
index b91861860e..fc6751209e 100644
--- a/go.sum
+++ b/go.sum
@@ -866,8 +866,8 @@ github.com/projectdiscovery/n3iwf v0.0.0-20230523120440-b8cd232ff1f5 h1:L/e8z8yw
github.com/projectdiscovery/n3iwf v0.0.0-20230523120440-b8cd232ff1f5/go.mod h1:pGW2ncnTxTxHtP9wzcIJAB+3/NMp6IiuQWd2NK7K+oc=
github.com/projectdiscovery/networkpolicy v0.0.9 h1:IrlDoYZagNNO8y+7iZeHT8k5izE+nek7TdtvEBwCxqk=
github.com/projectdiscovery/networkpolicy v0.0.9/go.mod h1:XFJ2Lnv8BE/ziQCFjBHMsH1w6VmkPiQtk+NlBpdMU7M=
-github.com/projectdiscovery/ratelimit v0.0.53 h1:1ci2zaVCtJ9daDi9IXjNwpDxl2InYq375role/xqUR4=
-github.com/projectdiscovery/ratelimit v0.0.53/go.mod h1:4kNYTL7xKLo1qMYHTREUCBqHyvykFF+XBWutTh9fkwg=
+github.com/projectdiscovery/ratelimit v0.0.56 h1:WliU7NvfMb5hK/IJjOFlWIXU1G7+QRylMhSybaSCTI8=
+github.com/projectdiscovery/ratelimit v0.0.56/go.mod h1:GbnAo+MbB4R/4jiOI1mH4KAJfovmrPnq4NElcI99fvs=
github.com/projectdiscovery/rawhttp v0.1.67 h1:HYzathMk3c8Y83hYjHM4GCBFbz/G+vABe0Lz6ajaowY=
github.com/projectdiscovery/rawhttp v0.1.67/go.mod h1:5viJ6odzc9ZuEFppj/E7HdX+u99FoYlvXnhHyTNc7N0=
github.com/projectdiscovery/rdap v0.9.1-0.20221108103045-9865884d1917 h1:m03X4gBVSorSzvmm0bFa7gDV4QNSOWPL/fgZ4kTXBxk=
From 3e6c62b03b4a471bfef8cc5bf63c1b7aff53eeb8 Mon Sep 17 00:00:00 2001
From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com>
Date: Mon, 23 Sep 2024 05:43:17 +0000
Subject: [PATCH 211/283] chore(deps): bump
github.com/projectdiscovery/useragent
Bumps [github.com/projectdiscovery/useragent](https://github.com/projectdiscovery/useragent) from 0.0.65 to 0.0.71.
- [Release notes](https://github.com/projectdiscovery/useragent/releases)
- [Commits](https://github.com/projectdiscovery/useragent/compare/v0.0.65...v0.0.71)
---
updated-dependencies:
- dependency-name: github.com/projectdiscovery/useragent
dependency-type: direct:production
update-type: version-update:semver-patch
...
Signed-off-by: dependabot[bot]
---
go.mod | 2 +-
go.sum | 4 ++--
2 files changed, 3 insertions(+), 3 deletions(-)
diff --git a/go.mod b/go.mod
index 0cdfc66be3..344e530191 100644
--- a/go.mod
+++ b/go.mod
@@ -96,7 +96,7 @@ require (
github.com/projectdiscovery/sarif v0.0.1
github.com/projectdiscovery/tlsx v1.1.7
github.com/projectdiscovery/uncover v1.0.9
- github.com/projectdiscovery/useragent v0.0.65
+ github.com/projectdiscovery/useragent v0.0.71
github.com/projectdiscovery/utils v0.2.10
github.com/projectdiscovery/wappalyzergo v0.1.18
github.com/redis/go-redis/v9 v9.1.0
diff --git a/go.sum b/go.sum
index b91861860e..8c68810570 100644
--- a/go.sum
+++ b/go.sum
@@ -884,8 +884,8 @@ github.com/projectdiscovery/tlsx v1.1.7 h1:eSsl/SmTDL/z2CMeSrbssk4f/9oOotMP1SgXl
github.com/projectdiscovery/tlsx v1.1.7/go.mod h1:g66QQ4/y4tLVjoGbzWIv+Q6xwFzxfJbEDx86Y1dYHDM=
github.com/projectdiscovery/uncover v1.0.9 h1:s5RbkD/V4r8QcPkys4gTTqMuRSgXq0JprejqLSopN9Y=
github.com/projectdiscovery/uncover v1.0.9/go.mod h1:2PUF3SpB5QNIJ8epaB2xbRzkPaxEAWRDm3Ir2ijt81U=
-github.com/projectdiscovery/useragent v0.0.65 h1:x78ZwWdqpzokOHxLITUXvq+ljkTKc19z3ILGtoV1N70=
-github.com/projectdiscovery/useragent v0.0.65/go.mod h1:deOP8YLJU6SCzM8k+K8PjkcOF4Ux0spqyO4ODZGIT4A=
+github.com/projectdiscovery/useragent v0.0.71 h1:Q02L3LV15ztOQ6FfmVSqVmOd5QhvzI+yAgYOc/32Nvg=
+github.com/projectdiscovery/useragent v0.0.71/go.mod h1:DHPruFLCvCvkd2qqPwwQZrP9sziv0lxQJ0R1rE1fa8E=
github.com/projectdiscovery/utils v0.2.10 h1:MFx9pAaSyYGjXNi46mdAD7ceSZ80E0AGeuBj8fXaAF4=
github.com/projectdiscovery/utils v0.2.10/go.mod h1:jvTckFUQxZSiJrhACUuU63tqtd0tdBu3ng3QUVm/Ce0=
github.com/projectdiscovery/wappalyzergo v0.1.18 h1:fFgETis0HcsNE7wREaUPYP45JqIyHgGorJaVp1RH7g4=
From 9983d7415c3da36d8fa588fde258516d280c7f57 Mon Sep 17 00:00:00 2001
From: Dwi Siswanto
Date: Mon, 23 Sep 2024 17:27:30 +0700
Subject: [PATCH 212/283] refactor(runner): adjust `max-host-error` if gt
`concurrency` (#5633)
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit
* refactor(common): use `ParseRequestURI` instead when `NormalizeCacheValue`
also it exports the method
Signed-off-by: Dwi Siswanto
* refactor(runner): adjust `max-host-error` if gt `concurrency`
Signed-off-by: Dwi Siswanto
* fix lint
* chore(runner): expose adjusted `max-host-error` value
Signed-off-by: Dwi Siswanto
---------
Signed-off-by: Dwi Siswanto
Co-authored-by: Doğan Can Bakır
---
internal/runner/runner.go | 11 ++++-
.../common/hosterrorscache/hosterrorscache.go | 42 ++++++++++++-------
.../hosterrorscache/hosterrorscache_test.go | 4 +-
3 files changed, 38 insertions(+), 19 deletions(-)
diff --git a/internal/runner/runner.go b/internal/runner/runner.go
index b36d8ed584..0b6da592d3 100644
--- a/internal/runner/runner.go
+++ b/internal/runner/runner.go
@@ -501,8 +501,17 @@ func (r *Runner) RunEnumeration() error {
}
if r.options.ShouldUseHostError() {
- cache := hosterrorscache.New(r.options.MaxHostError, hosterrorscache.DefaultMaxHostsCount, r.options.TrackError)
+ maxHostError := r.options.MaxHostError
+ if r.options.TemplateThreads > maxHostError {
+ gologger.Print().Msgf("[%v] The concurrency value is higher than max-host-error", r.colorizer.BrightYellow("WRN"))
+ gologger.Info().Msgf("Adjusting max-host-error to the concurrency value: %d", r.options.TemplateThreads)
+
+ maxHostError = r.options.TemplateThreads
+ }
+
+ cache := hosterrorscache.New(maxHostError, hosterrorscache.DefaultMaxHostsCount, r.options.TrackError)
cache.SetVerbose(r.options.Verbose)
+
r.hostErrors = cache
executorOpts.HostErrorsCache = cache
}
diff --git a/pkg/protocols/common/hosterrorscache/hosterrorscache.go b/pkg/protocols/common/hosterrorscache/hosterrorscache.go
index 1630e97c7d..bcfa27dbe4 100644
--- a/pkg/protocols/common/hosterrorscache/hosterrorscache.go
+++ b/pkg/protocols/common/hosterrorscache/hosterrorscache.go
@@ -75,24 +75,34 @@ func (c *Cache) Close() {
c.failedTargets.Purge()
}
-func (c *Cache) normalizeCacheValue(value string) string {
- finalValue := value
- if strings.HasPrefix(value, "http") {
- if parsed, err := url.Parse(value); err == nil {
- hostname := parsed.Host
- finalPort := parsed.Port()
- if finalPort == "" {
- if parsed.Scheme == "https" {
- finalPort = "443"
- } else {
- finalPort = "80"
- }
- hostname = net.JoinHostPort(parsed.Host, finalPort)
+// NormalizeCacheValue processes the input value and returns a normalized cache
+// value.
+func (c *Cache) NormalizeCacheValue(value string) string {
+ var normalizedValue string = value
+
+ u, err := url.ParseRequestURI(value)
+ if err != nil || u.Host == "" {
+ u, err2 := url.ParseRequestURI("https://" + value)
+ if err2 != nil {
+ return normalizedValue
+ }
+
+ normalizedValue = u.Host
+ } else {
+ port := u.Port()
+ if port == "" {
+ switch u.Scheme {
+ case "https":
+ normalizedValue = net.JoinHostPort(u.Host, "443")
+ case "http":
+ normalizedValue = net.JoinHostPort(u.Host, "80")
}
- finalValue = hostname
+ } else {
+ normalizedValue = u.Host
}
}
- return finalValue
+
+ return normalizedValue
}
// ErrUnresponsiveHost is returned when a host is unresponsive
@@ -166,7 +176,7 @@ func (c *Cache) GetKeyFromContext(ctx *contextargs.Context, err error) string {
address = tmp.String()
}
}
- finalValue := c.normalizeCacheValue(address)
+ finalValue := c.NormalizeCacheValue(address)
return finalValue
}
diff --git a/pkg/protocols/common/hosterrorscache/hosterrorscache_test.go b/pkg/protocols/common/hosterrorscache/hosterrorscache_test.go
index 112690d875..3c93177674 100644
--- a/pkg/protocols/common/hosterrorscache/hosterrorscache_test.go
+++ b/pkg/protocols/common/hosterrorscache/hosterrorscache_test.go
@@ -109,7 +109,7 @@ func TestCacheMarkFailedConcurrent(t *testing.T) {
// the cache is not atomic during items creation, so we pre-create them with counter to zero
for _, test := range tests {
- normalizedValue := cache.normalizeCacheValue(test.host)
+ normalizedValue := cache.NormalizeCacheValue(test.host)
newItem := &cacheItem{errors: atomic.Int32{}}
newItem.errors.Store(0)
_ = cache.failedTargets.Set(normalizedValue, newItem)
@@ -131,7 +131,7 @@ func TestCacheMarkFailedConcurrent(t *testing.T) {
for _, test := range tests {
require.True(t, cache.Check(newCtxArgs(test.host)))
- normalizedCacheValue := cache.normalizeCacheValue(test.host)
+ normalizedCacheValue := cache.NormalizeCacheValue(test.host)
failedTarget, err := cache.failedTargets.Get(normalizedCacheValue)
require.Nil(t, err)
require.NotNil(t, failedTarget)
From 828dac90025f803bd5621cd3f638189498b49d66 Mon Sep 17 00:00:00 2001
From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com>
Date: Mon, 23 Sep 2024 17:49:04 +0530
Subject: [PATCH 213/283] chore(deps): bump
github.com/projectdiscovery/retryabledns (#5657)
Bumps [github.com/projectdiscovery/retryabledns](https://github.com/projectdiscovery/retryabledns) from 1.0.74 to 1.0.77.
- [Release notes](https://github.com/projectdiscovery/retryabledns/releases)
- [Commits](https://github.com/projectdiscovery/retryabledns/compare/v1.0.74...v1.0.77)
---
updated-dependencies:
- dependency-name: github.com/projectdiscovery/retryabledns
dependency-type: direct:production
update-type: version-update:semver-patch
...
Signed-off-by: dependabot[bot]
Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com>
---
go.mod | 2 +-
go.sum | 4 ++--
2 files changed, 3 insertions(+), 3 deletions(-)
diff --git a/go.mod b/go.mod
index 73fc84aa9b..a4f29ff9b9 100644
--- a/go.mod
+++ b/go.mod
@@ -24,7 +24,7 @@ require (
github.com/projectdiscovery/hmap v0.0.58
github.com/projectdiscovery/interactsh v1.2.0
github.com/projectdiscovery/rawhttp v0.1.67
- github.com/projectdiscovery/retryabledns v1.0.76
+ github.com/projectdiscovery/retryabledns v1.0.77
github.com/projectdiscovery/retryablehttp-go v1.0.78
github.com/projectdiscovery/yamldoc-go v1.0.4
github.com/remeh/sizedwaitgroup v1.0.0
diff --git a/go.sum b/go.sum
index a51634bab3..4e5a765cf5 100644
--- a/go.sum
+++ b/go.sum
@@ -872,8 +872,8 @@ github.com/projectdiscovery/rawhttp v0.1.67 h1:HYzathMk3c8Y83hYjHM4GCBFbz/G+vABe
github.com/projectdiscovery/rawhttp v0.1.67/go.mod h1:5viJ6odzc9ZuEFppj/E7HdX+u99FoYlvXnhHyTNc7N0=
github.com/projectdiscovery/rdap v0.9.1-0.20221108103045-9865884d1917 h1:m03X4gBVSorSzvmm0bFa7gDV4QNSOWPL/fgZ4kTXBxk=
github.com/projectdiscovery/rdap v0.9.1-0.20221108103045-9865884d1917/go.mod h1:JxXtZC9e195awe7EynrcnBJmFoad/BNDzW9mzFkK8Sg=
-github.com/projectdiscovery/retryabledns v1.0.76 h1:6PyRCSW5SMUfE7uozZWckWuIDnBgLU/b1ofeWqeKONs=
-github.com/projectdiscovery/retryabledns v1.0.76/go.mod h1:vc4L2dng+nPLpQu7s0eqFc0OvyrZibuMNWFMA9bnN3w=
+github.com/projectdiscovery/retryabledns v1.0.77 h1:rCFSiTPNI7h9PP1uUvrmcq/6XVZVdxpha2H1ioArKpk=
+github.com/projectdiscovery/retryabledns v1.0.77/go.mod h1:ce1JTjAOxLujqITtA5VLlbhLRVubx+GETYasivapai4=
github.com/projectdiscovery/retryablehttp-go v1.0.78 h1:If7/XjCWk893YrnTMaW69TNMsfE1Er3i1SWOkWbEk4E=
github.com/projectdiscovery/retryablehttp-go v1.0.78/go.mod h1:NqzTdnGihSRkF9c/aXo+3qTJXEeNwnOuza6GrlPo9qw=
github.com/projectdiscovery/sarif v0.0.1 h1:C2Tyj0SGOKbCLgHrx83vaE6YkzXEVrMXYRGLkKCr/us=
From 6a561c64700d05ff2141860c768374d6ca1ea07d Mon Sep 17 00:00:00 2001
From: Ice3man
Date: Tue, 24 Sep 2024 18:43:35 +0530
Subject: [PATCH 214/283] feat: clone event in clustering to generate correct
failure events (#5653)
---
pkg/output/output.go | 10 ++++++++++
pkg/templates/cluster.go | 34 +++++++++++++++++++---------------
2 files changed, 29 insertions(+), 15 deletions(-)
diff --git a/pkg/output/output.go b/pkg/output/output.go
index 07c6f4c71f..fbc9f71306 100644
--- a/pkg/output/output.go
+++ b/pkg/output/output.go
@@ -5,6 +5,7 @@ import (
"fmt"
"io"
"log/slog"
+ "maps"
"os"
"path/filepath"
"regexp"
@@ -99,6 +100,15 @@ type InternalWrappedEvent struct {
InteractshMatched atomic.Bool
}
+func (iwe *InternalWrappedEvent) CloneShallow() *InternalWrappedEvent {
+ return &InternalWrappedEvent{
+ InternalEvent: maps.Clone(iwe.InternalEvent),
+ Results: nil,
+ OperatorsResult: nil,
+ UsesInteractsh: iwe.UsesInteractsh,
+ }
+}
+
func (iwe *InternalWrappedEvent) HasOperatorResult() bool {
iwe.RLock()
defer iwe.RUnlock()
diff --git a/pkg/templates/cluster.go b/pkg/templates/cluster.go
index a4a4ac4bad..8f1af96b74 100644
--- a/pkg/templates/cluster.go
+++ b/pkg/templates/cluster.go
@@ -251,23 +251,25 @@ func (e *ClusterExecuter) Execute(ctx *scan.ScanContext) (bool, error) {
event.InternalEvent = make(map[string]interface{})
}
for _, operator := range e.operators {
- result, matched := operator.operator.Execute(event.InternalEvent, e.requests.Match, e.requests.Extract, e.options.Options.Debug || e.options.Options.DebugResponse)
- event.InternalEvent["template-id"] = operator.templateID
- event.InternalEvent["template-path"] = operator.templatePath
- event.InternalEvent["template-info"] = operator.templateInfo
+ clonedEvent := event.CloneShallow()
+
+ result, matched := operator.operator.Execute(clonedEvent.InternalEvent, e.requests.Match, e.requests.Extract, e.options.Options.Debug || e.options.Options.DebugResponse)
+ clonedEvent.InternalEvent["template-id"] = operator.templateID
+ clonedEvent.InternalEvent["template-path"] = operator.templatePath
+ clonedEvent.InternalEvent["template-info"] = operator.templateInfo
if result == nil && !matched && e.options.Options.MatcherStatus {
- if err := e.options.Output.WriteFailure(event); err != nil {
+ if err := e.options.Output.WriteFailure(clonedEvent); err != nil {
gologger.Warning().Msgf("Could not write failure event to output: %s\n", err)
}
continue
}
if matched && result != nil {
- event.OperatorsResult = result
- event.Results = e.requests.MakeResultEvent(event)
+ clonedEvent.OperatorsResult = result
+ clonedEvent.Results = e.requests.MakeResultEvent(clonedEvent)
results = true
- _ = writer.WriteResult(event, e.options.Output, e.options.Progress, e.options.IssuesClient)
+ _ = writer.WriteResult(clonedEvent, e.options.Output, e.options.Progress, e.options.IssuesClient)
}
}
})
@@ -290,14 +292,16 @@ func (e *ClusterExecuter) ExecuteWithResults(ctx *scan.ScanContext) ([]*output.R
}
err := e.requests.ExecuteWithResults(inputItem, dynamicValues, nil, func(event *output.InternalWrappedEvent) {
for _, operator := range e.operators {
- result, matched := operator.operator.Execute(event.InternalEvent, e.requests.Match, e.requests.Extract, e.options.Options.Debug || e.options.Options.DebugResponse)
+ clonedEvent := event.CloneShallow()
+
+ result, matched := operator.operator.Execute(clonedEvent.InternalEvent, e.requests.Match, e.requests.Extract, e.options.Options.Debug || e.options.Options.DebugResponse)
if matched && result != nil {
- event.OperatorsResult = result
- event.InternalEvent["template-id"] = operator.templateID
- event.InternalEvent["template-path"] = operator.templatePath
- event.InternalEvent["template-info"] = operator.templateInfo
- event.Results = e.requests.MakeResultEvent(event)
- scanCtx.LogEvent(event)
+ clonedEvent.OperatorsResult = result
+ clonedEvent.InternalEvent["template-id"] = operator.templateID
+ clonedEvent.InternalEvent["template-path"] = operator.templatePath
+ clonedEvent.InternalEvent["template-info"] = operator.templateInfo
+ clonedEvent.Results = e.requests.MakeResultEvent(clonedEvent)
+ scanCtx.LogEvent(clonedEvent)
}
}
})
From c93a2b1ba412adb7aa353d97c0cdadd581ff6602 Mon Sep 17 00:00:00 2001
From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com>
Date: Tue, 24 Sep 2024 18:44:01 +0530
Subject: [PATCH 215/283] chore(deps): bump github.com/projectdiscovery/utils
from 0.2.8 to 0.2.11 (#5660)
Bumps [github.com/projectdiscovery/utils](https://github.com/projectdiscovery/utils) from 0.2.8 to 0.2.11.
- [Release notes](https://github.com/projectdiscovery/utils/releases)
- [Changelog](https://github.com/projectdiscovery/utils/blob/main/CHANGELOG.md)
- [Commits](https://github.com/projectdiscovery/utils/compare/v0.2.8...v0.2.11)
---
updated-dependencies:
- dependency-name: github.com/projectdiscovery/utils
dependency-type: direct:production
update-type: version-update:semver-patch
...
Signed-off-by: dependabot[bot]
Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com>
---
go.mod | 35 ++++++++++++-----------
go.sum | 89 ++++++++++++++++++++++++++++++++--------------------------
2 files changed, 68 insertions(+), 56 deletions(-)
diff --git a/go.mod b/go.mod
index a4f29ff9b9..f2adacca21 100644
--- a/go.mod
+++ b/go.mod
@@ -21,7 +21,7 @@ require (
github.com/pkg/errors v0.9.1
github.com/projectdiscovery/clistats v0.1.1
github.com/projectdiscovery/fastdialer v0.2.9
- github.com/projectdiscovery/hmap v0.0.58
+ github.com/projectdiscovery/hmap v0.0.59
github.com/projectdiscovery/interactsh v1.2.0
github.com/projectdiscovery/rawhttp v0.1.67
github.com/projectdiscovery/retryabledns v1.0.77
@@ -38,9 +38,9 @@ require (
github.com/weppos/publicsuffix-go v0.30.2
github.com/xanzy/go-gitlab v0.107.0
go.uber.org/multierr v1.11.0
- golang.org/x/net v0.26.0
+ golang.org/x/net v0.29.0
golang.org/x/oauth2 v0.22.0
- golang.org/x/text v0.16.0
+ golang.org/x/text v0.18.0
gopkg.in/yaml.v2 v2.4.0
)
@@ -59,7 +59,7 @@ require (
github.com/aws/aws-sdk-go-v2/feature/s3/manager v1.11.72
github.com/aws/aws-sdk-go-v2/service/s3 v1.37.0
github.com/cespare/xxhash v1.1.0
- github.com/charmbracelet/glamour v0.6.0
+ github.com/charmbracelet/glamour v0.8.0
github.com/clbanning/mxj/v2 v2.7.0
github.com/ditashi/jsbeautifier-go v0.0.0-20141206144643-2520a8026a9c
github.com/docker/go-units v0.5.0
@@ -97,7 +97,7 @@ require (
github.com/projectdiscovery/tlsx v1.1.7
github.com/projectdiscovery/uncover v1.0.9
github.com/projectdiscovery/useragent v0.0.71
- github.com/projectdiscovery/utils v0.2.10
+ github.com/projectdiscovery/utils v0.2.11
github.com/projectdiscovery/wappalyzergo v0.1.18
github.com/redis/go-redis/v9 v9.1.0
github.com/seh-msft/burpxml v1.0.1
@@ -105,7 +105,7 @@ require (
github.com/stretchr/testify v1.9.0
github.com/tarunKoyalwar/goleak v0.0.0-20240429141123-0efa90dbdcf9
github.com/zmap/zgrab2 v0.1.8-0.20230806160807-97ba87c0e706
- golang.org/x/term v0.21.0
+ golang.org/x/term v0.24.0
gopkg.in/yaml.v3 v3.0.1
moul.io/http2curl v1.0.0
)
@@ -120,6 +120,7 @@ require (
github.com/AzureAD/microsoft-authentication-library-for-go v1.2.2 // indirect
github.com/Nvveen/Gotty v0.0.0-20120604004816-cd527374f1e5 // indirect
github.com/VividCortex/ewma v1.2.0 // indirect
+ github.com/alecthomas/chroma/v2 v2.14.0 // indirect
github.com/andybalholm/brotli v1.1.0 // indirect
github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.4.10 // indirect
github.com/aws/aws-sdk-go-v2/internal/v4a v1.0.27 // indirect
@@ -134,6 +135,8 @@ require (
github.com/bytedance/sonic v1.9.1 // indirect
github.com/cenkalti/backoff/v4 v4.2.1 // indirect
github.com/cespare/xxhash/v2 v2.2.0 // indirect
+ github.com/charmbracelet/lipgloss v0.13.0 // indirect
+ github.com/charmbracelet/x/ansi v0.3.2 // indirect
github.com/cheggaaa/pb/v3 v3.1.4 // indirect
github.com/chenzhuoyu/base64x v0.0.0-20221115062448-fe3a3abad311 // indirect
github.com/cloudflare/cfssl v1.6.4 // indirect
@@ -142,7 +145,7 @@ require (
github.com/cyphar/filepath-securejoin v0.2.4 // indirect
github.com/davidmz/go-pageant v1.0.2 // indirect
github.com/dgryski/go-rendezvous v0.0.0-20200823014737-9f7001d12a5f // indirect
- github.com/dlclark/regexp2 v1.11.0 // indirect
+ github.com/dlclark/regexp2 v1.11.4 // indirect
github.com/docker/cli v24.0.5+incompatible // indirect
github.com/docker/docker v24.0.9+incompatible // indirect
github.com/docker/go-connections v0.4.0 // indirect
@@ -193,7 +196,7 @@ require (
github.com/moby/term v0.5.0 // indirect
github.com/mohae/deepcopy v0.0.0-20170929034955-c48cc78d4826 // indirect
github.com/muesli/reflow v0.3.0 // indirect
- github.com/muesli/termenv v0.15.1 // indirect
+ github.com/muesli/termenv v0.15.3-0.20240618155329-98d742f6907a // indirect
github.com/opencontainers/go-digest v1.0.0 // indirect
github.com/opencontainers/image-spec v1.0.2 // indirect
github.com/opencontainers/runc v1.1.14 // indirect
@@ -230,13 +233,13 @@ require (
github.com/xeipuuv/gojsonschema v1.2.0 // indirect
github.com/ysmood/fetchup v0.2.3 // indirect
github.com/ysmood/got v0.34.1 // indirect
- github.com/yuin/goldmark v1.5.4 // indirect
- github.com/yuin/goldmark-emoji v1.0.1 // indirect
+ github.com/yuin/goldmark v1.7.4 // indirect
+ github.com/yuin/goldmark-emoji v1.0.3 // indirect
github.com/zcalusic/sysinfo v1.0.2 // indirect
github.com/zeebo/blake3 v0.2.3 // indirect
go.uber.org/goleak v1.3.0 // indirect
golang.org/x/arch v0.3.0 // indirect
- golang.org/x/sync v0.7.0 // indirect
+ golang.org/x/sync v0.8.0 // indirect
gopkg.in/djherbis/times.v1 v1.3.0 // indirect
mellium.im/sasl v0.3.1 // indirect
)
@@ -278,16 +281,16 @@ require (
github.com/lor00x/goldap v0.0.0-20180618054307-a546dffdd1a3 // indirect
github.com/lufia/plan9stats v0.0.0-20211012122336-39d0f177ccd0 // indirect
github.com/mattn/go-isatty v0.0.20 // indirect
- github.com/mattn/go-runewidth v0.0.15 // indirect
+ github.com/mattn/go-runewidth v0.0.16 // indirect
github.com/mholt/acmez v1.2.0 // indirect
- github.com/microcosm-cc/bluemonday v1.0.26 // indirect
+ github.com/microcosm-cc/bluemonday v1.0.27 // indirect
github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect
github.com/modern-go/reflect2 v1.0.2 // indirect
github.com/pmezard/go-difflib v1.0.0 // indirect
github.com/power-devops/perfstat v0.0.0-20210106213030-5aafc221ea8c // indirect
github.com/projectdiscovery/blackrock v0.0.1 // indirect
github.com/projectdiscovery/networkpolicy v0.0.9
- github.com/rivo/uniseg v0.4.6 // indirect
+ github.com/rivo/uniseg v0.4.7 // indirect
github.com/saintfish/chardet v0.0.0-20230101081208-5e3ef4b5456d // indirect
github.com/tklauser/go-sysconf v0.3.12 // indirect
github.com/tklauser/numcpus v0.6.1 // indirect
@@ -303,10 +306,10 @@ require (
go.etcd.io/bbolt v1.3.10 // indirect
go.uber.org/zap v1.25.0 // indirect
goftp.io/server/v2 v2.0.1 // indirect
- golang.org/x/crypto v0.24.0 // indirect
+ golang.org/x/crypto v0.27.0 // indirect
golang.org/x/exp v0.0.0-20240506185415-9bf2ced13842
golang.org/x/mod v0.17.0 // indirect
- golang.org/x/sys v0.21.0 // indirect
+ golang.org/x/sys v0.25.0 // indirect
golang.org/x/time v0.6.0 // indirect
golang.org/x/tools v0.21.1-0.20240508182429-e35e4ccd0d2d
google.golang.org/protobuf v1.34.2 // indirect
diff --git a/go.sum b/go.sum
index 4e5a765cf5..da71e9b287 100644
--- a/go.sum
+++ b/go.sum
@@ -93,9 +93,15 @@ github.com/VividCortex/gohistogram v1.0.0/go.mod h1:Pf5mBqqDxYaXu3hDrrU+w6nw50o/
github.com/afex/hystrix-go v0.0.0-20180502004556-fa1af6a1f4f5/go.mod h1:SkGFH1ia65gfNATL8TAiHDNxPzPdmEL5uirI2Uyuz6c=
github.com/akrylysov/pogreb v0.10.2 h1:e6PxmeyEhWyi2AKOBIJzAEi4HkiC+lKyCocRGlnDi78=
github.com/akrylysov/pogreb v0.10.2/go.mod h1:pNs6QmpQ1UlTJKDezuRWmaqkgUE2TuU0YTWyqJZ7+lI=
+github.com/alecthomas/assert/v2 v2.7.0 h1:QtqSACNS3tF7oasA8CU6A6sXZSBDqnm7RfpLl9bZqbE=
+github.com/alecthomas/assert/v2 v2.7.0/go.mod h1:Bze95FyfUr7x34QZrjL+XP+0qgp/zg8yS+TtBj1WA3k=
github.com/alecthomas/chroma v0.10.0 h1:7XDcGkCQopCNKjZHfYrNLraA+M7e0fMiJ/Mfikbfjek=
github.com/alecthomas/chroma v0.10.0/go.mod h1:jtJATyUxlIORhUOFNA9NZDWGAQ8wpxQQqNSB4rjA/1s=
+github.com/alecthomas/chroma/v2 v2.14.0 h1:R3+wzpnUArGcQz7fCETQBzO5n9IMNi13iIs46aU4V9E=
+github.com/alecthomas/chroma/v2 v2.14.0/go.mod h1:QolEbTfmUHIMVpBqxeDnNBj2uoeI4EbYP4i6n68SG4I=
github.com/alecthomas/kingpin/v2 v2.3.1/go.mod h1:oYL5vtsvEHZGHxU7DMp32Dvx+qL+ptGn6lWaot2vCNE=
+github.com/alecthomas/repr v0.4.0 h1:GhI2A8MACjfegCPVq9f1FLvIBS+DrQ2KQBFZP1iFzXc=
+github.com/alecthomas/repr v0.4.0/go.mod h1:Fr0507jx4eOXV7AlPV6AVZLYrLIuIeSOWtW57eE/O/4=
github.com/alecthomas/template v0.0.0-20160405071501-a0175ee3bccc/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc=
github.com/alecthomas/template v0.0.0-20190718012654-fb15b899a751 h1:JYp7IbQjafoB+tBA3gMyHYHrpOtNuDiK/uB5uXxq5wM=
github.com/alecthomas/template v0.0.0-20190718012654-fb15b899a751/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc=
@@ -174,9 +180,10 @@ github.com/aws/aws-sdk-go-v2/service/sts v1.19.3 h1:e5mnydVdCVWxP+5rPAGi2PYxC7u2
github.com/aws/aws-sdk-go-v2/service/sts v1.19.3/go.mod h1:yVGZA1CPkmUhBdA039jXNJJG7/6t+G+EBWmFq23xqnY=
github.com/aws/smithy-go v1.13.5 h1:hgz0X/DX0dGqTYpGALqXJoRKRj5oQ7150i5FdTePzO8=
github.com/aws/smithy-go v1.13.5/go.mod h1:Tg+OJXh4MB2R/uN61Ko2f6hTZwB/ZYGOtib8J3gBHzA=
-github.com/aymanbagabas/go-osc52 v1.0.3/go.mod h1:zT8H+Rk4VSabYN90pWyugflM3ZhpTZNC7cASDfUCdT4=
github.com/aymanbagabas/go-osc52/v2 v2.0.1 h1:HwpRHbFMcZLEVr42D4p7XBqjyuxQH5SMiErDT4WkJ2k=
github.com/aymanbagabas/go-osc52/v2 v2.0.1/go.mod h1:uYgXzlJ7ZpABp8OJ+exZzJJhRNQ2ASbcXHWsFqH8hp8=
+github.com/aymanbagabas/go-udiff v0.2.0 h1:TK0fH4MteXUDspT88n8CKzvK0X9O2xu9yQjWpi6yML8=
+github.com/aymanbagabas/go-udiff v0.2.0/go.mod h1:RE4Ex0qsGkTAJoQdQQCA0uG+nAzJO/pI/QwceO5fgrA=
github.com/aymerick/douceur v0.2.0 h1:Mv+mAeH1Q+n9Fr+oyamOlAkUNPWPlA8PPGR0QAaYuPk=
github.com/aymerick/douceur v0.2.0/go.mod h1:wlT5vV2O3h55X9m7iVYN0TBM0NH/MmbLnd30/FjWUq4=
github.com/bahlo/generic-list-go v0.2.0 h1:5sz/EEAK+ls5wF+NeqDpk5+iNdMDXrh3z3nPnH1Wvgk=
@@ -217,8 +224,14 @@ github.com/cespare/xxhash/v2 v2.1.1/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XL
github.com/cespare/xxhash/v2 v2.1.2/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs=
github.com/cespare/xxhash/v2 v2.2.0 h1:DC2CZ1Ep5Y4k3ZQ899DldepgrayRUGE6BBZ/cd9Cj44=
github.com/cespare/xxhash/v2 v2.2.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs=
-github.com/charmbracelet/glamour v0.6.0 h1:wi8fse3Y7nfcabbbDuwolqTqMQPMnVPeZhDM273bISc=
-github.com/charmbracelet/glamour v0.6.0/go.mod h1:taqWV4swIMMbWALc0m7AfE9JkPSU8om2538k9ITBxOc=
+github.com/charmbracelet/glamour v0.8.0 h1:tPrjL3aRcQbn++7t18wOpgLyl8wrOHUEDS7IZ68QtZs=
+github.com/charmbracelet/glamour v0.8.0/go.mod h1:ViRgmKkf3u5S7uakt2czJ272WSg2ZenlYEZXT2x7Bjw=
+github.com/charmbracelet/lipgloss v0.13.0 h1:4X3PPeoWEDCMvzDvGmTajSyYPcZM4+y8sCA/SsA3cjw=
+github.com/charmbracelet/lipgloss v0.13.0/go.mod h1:nw4zy0SBX/F/eAO1cWdcvy6qnkDUxr8Lw7dvFrAIbbY=
+github.com/charmbracelet/x/ansi v0.3.2 h1:wsEwgAN+C9U06l9dCVMX0/L3x7ptvY1qmjMwyfE6USY=
+github.com/charmbracelet/x/ansi v0.3.2/go.mod h1:dk73KoMTT5AX5BsX0KrqhsTqAnhZZoCBjs7dGWp4Ktw=
+github.com/charmbracelet/x/exp/golden v0.0.0-20240806155701-69247e0abc2a h1:G99klV19u0QnhiizODirwVksQB91TJKV/UaTnACcG30=
+github.com/charmbracelet/x/exp/golden v0.0.0-20240806155701-69247e0abc2a/go.mod h1:wDlXFlCrmJ8J+swcL/MnGUuYnqgQdW9rhSD61oNMb6U=
github.com/cheggaaa/pb/v3 v3.1.4 h1:DN8j4TVVdKu3WxVwcRKu0sG00IIU6FewoABZzXbRQeo=
github.com/cheggaaa/pb/v3 v3.1.4/go.mod h1:6wVjILNBaXMs8c21qRiaUM8BR82erfgau1DQ4iUXmSA=
github.com/chenzhuoyu/base64x v0.0.0-20211019084208-fb5309c8db06/go.mod h1:DH46F32mSOjUmXrMHnKwZdA8wcEefY7UVqBKYGjpdQY=
@@ -271,8 +284,8 @@ github.com/ditashi/jsbeautifier-go v0.0.0-20141206144643-2520a8026a9c/go.mod h1:
github.com/dlclark/regexp2 v1.4.0/go.mod h1:2pZnwuY/m+8K6iRw6wQdMtk+rH5tNGR1i55kozfMjCc=
github.com/dlclark/regexp2 v1.4.1-0.20201116162257-a2a8dda75c91/go.mod h1:2pZnwuY/m+8K6iRw6wQdMtk+rH5tNGR1i55kozfMjCc=
github.com/dlclark/regexp2 v1.7.0/go.mod h1:DHkYz0B9wPfa6wondMfaivmHpzrQ3v9q8cnmRbL6yW8=
-github.com/dlclark/regexp2 v1.11.0 h1:G/nrcoOa7ZXlpoa/91N3X7mM3r8eIlMBBJZvsz/mxKI=
-github.com/dlclark/regexp2 v1.11.0/go.mod h1:DHkYz0B9wPfa6wondMfaivmHpzrQ3v9q8cnmRbL6yW8=
+github.com/dlclark/regexp2 v1.11.4 h1:rPYF9/LECdNymJufQKmri9gV604RvvABwgOA8un7yAo=
+github.com/dlclark/regexp2 v1.11.4/go.mod h1:DHkYz0B9wPfa6wondMfaivmHpzrQ3v9q8cnmRbL6yW8=
github.com/docker/cli v24.0.5+incompatible h1:WeBimjvS0eKdH4Ygx+ihVq1Q++xg36M/rMi4aXAvodc=
github.com/docker/cli v24.0.5+incompatible/go.mod h1:JLrzqnKDaYBop7H2jaqPtU4hHvMKP+vjCwu2uszcLI8=
github.com/docker/docker v24.0.9+incompatible h1:HPGzNmwfLZWdxHqK9/II92pyi1EpYKsAqcl4G0Of9v0=
@@ -508,7 +521,6 @@ github.com/googleapis/gax-go/v2 v2.0.5/go.mod h1:DWXyrwAJ9X0FpwwEdw+IPEYBICEFu5m
github.com/gopherjs/gopherjs v0.0.0-20181017120253-0766667cb4d1 h1:EGx4pi6eqNxGaHF6qqu48+N2wcFQ5qg5FXgOdqsJ5d8=
github.com/gopherjs/gopherjs v0.0.0-20181017120253-0766667cb4d1/go.mod h1:wJfORRmW1u3UXTncJ5qlYoELFm8eSnnEO6hX4iZ3EWY=
github.com/gorilla/context v1.1.1/go.mod h1:kBGZzfjB9CEq2AlWe17Uuf7NDRt0dE0s8S51q0aT7Yg=
-github.com/gorilla/css v1.0.0/go.mod h1:Dn721qIggHpt4+EFCcTLTU/vk5ySda2ReITrtgBl60c=
github.com/gorilla/css v1.0.1 h1:ntNaBIghp6JmvWnxbZKANoLyuXTPZ4cAMlo6RyhlbO8=
github.com/gorilla/css v1.0.1/go.mod h1:BvnYkspnSzMmwRK+b8/xgNPLiIuNZr6vbZBTPQ2A3b0=
github.com/gorilla/mux v1.6.2/go.mod h1:1lud6UwP+6orDFRuTfBEV8e9/aOM/c4fVVCaMa2zaAs=
@@ -559,6 +571,8 @@ github.com/hbakhtiyor/strsim v0.0.0-20190107154042-4d2bbb273edf h1:umfGUaWdFP2s6
github.com/hbakhtiyor/strsim v0.0.0-20190107154042-4d2bbb273edf/go.mod h1:V99KdStnMHZsvVOwIvhfcUzYgYkRZeQWUtumtL+SKxA=
github.com/hdm/jarm-go v0.0.7 h1:Eq0geenHrBSYuKrdVhrBdMMzOmA+CAMLzN2WrF3eL6A=
github.com/hdm/jarm-go v0.0.7/go.mod h1:kinGoS0+Sdn1Rr54OtanET5E5n7AlD6T6CrJAKDjJSQ=
+github.com/hexops/gotextdiff v1.0.3 h1:gitA9+qJrrTCsiCl7+kh75nPqQt1cx4ZkudSTLoUqJM=
+github.com/hexops/gotextdiff v1.0.3/go.mod h1:pSWU5MAI3yDq+fZBTazCSJysOMbxWL1BSow5/V2vxeg=
github.com/hpcloud/tail v1.0.0/go.mod h1:ab1qPbhIpdTxEkNHXyeSf5vhxWSCs/tWer42PpOxQnU=
github.com/hudl/fargo v1.3.0/go.mod h1:y3CKSmjA+wD2gak7sUSXTAoopbhU08POFhmITJgmKTg=
github.com/ianlancetaylor/demangle v0.0.0-20181102032728-5e5cf60278f6/go.mod h1:aSSvb/t6k1mPoxDqO4vJh6VOCGPwU4O0C2/Eqndh1Sc=
@@ -695,9 +709,8 @@ github.com/mattn/go-isatty v0.0.20/go.mod h1:W+V8PltTTMOvKvAeJH7IuucS94S2C6jfK/D
github.com/mattn/go-runewidth v0.0.2/go.mod h1:LwmH8dsx7+W8Uxz3IHJYH5QSwggIsqBzpuz5H//U1FU=
github.com/mattn/go-runewidth v0.0.9/go.mod h1:H031xJmbD/WCDINGzjvQ9THkh0rPKHF+m2gUSrubnMI=
github.com/mattn/go-runewidth v0.0.12/go.mod h1:RAqKPSqVFrSLVXbA8x7dzmKdmGzieGRCM46jaSJTDAk=
-github.com/mattn/go-runewidth v0.0.14/go.mod h1:Jdepj2loyihRzMpdS35Xk/zdY8IAYHsh153qUoGf23w=
-github.com/mattn/go-runewidth v0.0.15 h1:UNAjwbU9l54TA3KzvqLGxwWjHmMgBUVhBiTjelZgg3U=
-github.com/mattn/go-runewidth v0.0.15/go.mod h1:Jdepj2loyihRzMpdS35Xk/zdY8IAYHsh153qUoGf23w=
+github.com/mattn/go-runewidth v0.0.16 h1:E5ScNMtiwvlvB5paMFdw9p4kSQzbXFikJ5SQO6TULQc=
+github.com/mattn/go-runewidth v0.0.16/go.mod h1:Jdepj2loyihRzMpdS35Xk/zdY8IAYHsh153qUoGf23w=
github.com/mattn/go-sqlite3 v1.14.22 h1:2gZY6PC6kBnID23Tichd1K+Z0oS6nE/XwU+Vz/5o4kU=
github.com/mattn/go-sqlite3 v1.14.22/go.mod h1:Uh1q+B4BYcTPb+yiD3kU8Ct7aC0hY9fxUwlHK0RXw+Y=
github.com/matttproud/golang_protobuf_extensions v1.0.1/go.mod h1:D8He9yQNgCq6Z5Ld7szi9bcBfOoFv/3dc6xSMkL2PC0=
@@ -708,9 +721,8 @@ github.com/mholt/archiver v3.1.1+incompatible h1:1dCVxuqs0dJseYEhi5pl7MYPH9zDa1w
github.com/mholt/archiver v3.1.1+incompatible/go.mod h1:Dh2dOXnSdiLxRiPoVfIr/fI1TwETms9B8CTWfeh7ROU=
github.com/mholt/archiver/v3 v3.5.1 h1:rDjOBX9JSF5BvoJGvjqK479aL70qh9DIpZCl+k7Clwo=
github.com/mholt/archiver/v3 v3.5.1/go.mod h1:e3dqJ7H78uzsRSEACH1joayhuSyhnonssnDhppzS1L4=
-github.com/microcosm-cc/bluemonday v1.0.21/go.mod h1:ytNkv4RrDrLJ2pqlsSI46O6IVXmZOBBD4SaJyDwwTkM=
-github.com/microcosm-cc/bluemonday v1.0.26 h1:xbqSvqzQMeEHCqMi64VAs4d8uy6Mequs3rQ0k/Khz58=
-github.com/microcosm-cc/bluemonday v1.0.26/go.mod h1:JyzOCs9gkyQyjs+6h10UEVSe02CGwkhd72Xdqh78TWs=
+github.com/microcosm-cc/bluemonday v1.0.27 h1:MpEUotklkwCSLeH+Qdx1VJgNqLlpY2KXwXFM08ygZfk=
+github.com/microcosm-cc/bluemonday v1.0.27/go.mod h1:jFi9vgW+H7c3V0lb6nR74Ib/DIB5OBs92Dimizgw2cA=
github.com/microsoft/go-mssqldb v1.6.0 h1:mM3gYdVwEPFrlg/Dvr2DNVEgYFG7L42l+dGc67NNNpc=
github.com/microsoft/go-mssqldb v1.6.0/go.mod h1:00mDtPbeQCRGC1HwOOR5K/gr30P1NcEG0vx6Kbv2aJU=
github.com/miekg/dns v1.0.14/go.mod h1:W1PPwlIAgtquWBMBEV9nkV9Cazfe8ScdGz/Lj7v3Nrg=
@@ -747,9 +759,8 @@ github.com/mreiferson/go-httpclient v0.0.0-20160630210159-31f0106b4474/go.mod h1
github.com/mreiferson/go-httpclient v0.0.0-20201222173833-5e475fde3a4d/go.mod h1:OQA4XLvDbMgS8P0CevmM4m9Q3Jq4phKUzcocxuGJ5m8=
github.com/muesli/reflow v0.3.0 h1:IFsN6K9NfGtjeggFP+68I4chLZV2yIKsXJFNZ+eWh6s=
github.com/muesli/reflow v0.3.0/go.mod h1:pbwTDkVPibjO2kyvBQRBxTWEEGDGq0FlB1BIKtnHY/8=
-github.com/muesli/termenv v0.13.0/go.mod h1:sP1+uffeLaEYpyOTb8pLCUctGcGLnoFjSn4YJK5e2bc=
-github.com/muesli/termenv v0.15.1 h1:UzuTb/+hhlBugQz28rpzey4ZuKcZ03MeKsoG7IJZIxs=
-github.com/muesli/termenv v0.15.1/go.mod h1:HeAQPTzpfs016yGtA4g00CsdYnVLJvxsS4ANqrZs2sQ=
+github.com/muesli/termenv v0.15.3-0.20240618155329-98d742f6907a h1:2MaM6YC3mGu54x+RKAA6JiFFHlHDY1UbkxqppT7wYOg=
+github.com/muesli/termenv v0.15.3-0.20240618155329-98d742f6907a/go.mod h1:hxSnBBYLK21Vtq/PHd0S2FYCxBXzBua8ov5s1RobyRQ=
github.com/mwitkow/go-conntrack v0.0.0-20161129095857-cc309e4a2223/go.mod h1:qRWi+5nqEBWmkhHvq77mSJWrCKwh8bxhgT7d/eI7P4U=
github.com/mwitkow/go-conntrack v0.0.0-20190716064945-2f068394615f/go.mod h1:qRWi+5nqEBWmkhHvq77mSJWrCKwh8bxhgT7d/eI7P4U=
github.com/nats-io/jwt v0.3.0/go.mod h1:fRYCDE99xlTsqUzISS1Bi75UBJ6ljOJQOAAu5VglpSg=
@@ -850,8 +861,8 @@ github.com/projectdiscovery/gostruct v0.0.2 h1:s8gP8ApugGM4go1pA+sVlPDXaWqNP5BBD
github.com/projectdiscovery/gostruct v0.0.2/go.mod h1:H86peL4HKwMXcQQtEa6lmC8FuD9XFt6gkNR0B/Mu5PE=
github.com/projectdiscovery/gozero v0.0.2 h1:8fJeaCjxL9tpm33uG/RsCQs6HGM/NE6eA3cjkilRQ+E=
github.com/projectdiscovery/gozero v0.0.2/go.mod h1:d8bZvDWW07LWNYWrwjZ4OO1I0cpkfqaysyDfSs9ibK8=
-github.com/projectdiscovery/hmap v0.0.58 h1:SoXmnmYS2egPSRgFKgUhHozu8QvPIUKAuoDpuii9jkw=
-github.com/projectdiscovery/hmap v0.0.58/go.mod h1:nQTelqkgxU6vuuU+qmQloMrDBxYZMt2TTO0fV86yXN4=
+github.com/projectdiscovery/hmap v0.0.59 h1:xWCr/GY2QJanFzwKydh/EkGdOKM4iAcN9hQvvCMgO6A=
+github.com/projectdiscovery/hmap v0.0.59/go.mod h1:uHhhnPmvq9qXvCjBSQXCBAlmA1r8JGufP775IkBSbgs=
github.com/projectdiscovery/httpx v1.6.8 h1:k0Y5g3ue/7QbDP0+LykIxp/VhPDLfau3UEUyuxtP7qE=
github.com/projectdiscovery/httpx v1.6.8/go.mod h1:7BIsDxyRwkBjthqFmEajXrA5f3yb4tlVfLmpNdf0ZXA=
github.com/projectdiscovery/interactsh v1.2.0 h1:Al6jHiR+Usl9egYJDLJaWNHOcH8Rugk8gWMasc8Cmw8=
@@ -886,8 +897,8 @@ github.com/projectdiscovery/uncover v1.0.9 h1:s5RbkD/V4r8QcPkys4gTTqMuRSgXq0Jpre
github.com/projectdiscovery/uncover v1.0.9/go.mod h1:2PUF3SpB5QNIJ8epaB2xbRzkPaxEAWRDm3Ir2ijt81U=
github.com/projectdiscovery/useragent v0.0.71 h1:Q02L3LV15ztOQ6FfmVSqVmOd5QhvzI+yAgYOc/32Nvg=
github.com/projectdiscovery/useragent v0.0.71/go.mod h1:DHPruFLCvCvkd2qqPwwQZrP9sziv0lxQJ0R1rE1fa8E=
-github.com/projectdiscovery/utils v0.2.10 h1:MFx9pAaSyYGjXNi46mdAD7ceSZ80E0AGeuBj8fXaAF4=
-github.com/projectdiscovery/utils v0.2.10/go.mod h1:jvTckFUQxZSiJrhACUuU63tqtd0tdBu3ng3QUVm/Ce0=
+github.com/projectdiscovery/utils v0.2.11 h1:TO7fBG5QI256sn1YuTD87yn4+4OjGJ2wT1772uEnp4Q=
+github.com/projectdiscovery/utils v0.2.11/go.mod h1:W0E74DWkKxlcyKS5XwcAwiob7+smoszPPi1NgX3vZyk=
github.com/projectdiscovery/wappalyzergo v0.1.18 h1:fFgETis0HcsNE7wREaUPYP45JqIyHgGorJaVp1RH7g4=
github.com/projectdiscovery/wappalyzergo v0.1.18/go.mod h1:/hzgxkBFTMe2wDbA93nFfoMjULw7/vIZ9QPSAnCgUa8=
github.com/projectdiscovery/yamldoc-go v1.0.4 h1:eZoESapnMw6WAHiVgRwNqvbJEfNHEH148uthhFbG5jE=
@@ -933,8 +944,8 @@ github.com/remeh/sizedwaitgroup v1.0.0 h1:VNGGFwNo/R5+MJBf6yrsr110p0m4/OX4S3DCy7
github.com/remeh/sizedwaitgroup v1.0.0/go.mod h1:3j2R4OIe/SeS6YDhICBy22RWjJC5eNCJ1V+9+NVNYlo=
github.com/rivo/uniseg v0.1.0/go.mod h1:J6wj4VEh+S6ZtnVlnTBMWIodfgj8LQOQFoIToxlJtxc=
github.com/rivo/uniseg v0.2.0/go.mod h1:J6wj4VEh+S6ZtnVlnTBMWIodfgj8LQOQFoIToxlJtxc=
-github.com/rivo/uniseg v0.4.6 h1:Sovz9sDSwbOz9tgUy8JpT+KgCkPYJEN/oYzlJiYTNLg=
-github.com/rivo/uniseg v0.4.6/go.mod h1:FN3SvrM+Zdj16jyLfmOkMNblXMcoc8DfTHruCPUcx88=
+github.com/rivo/uniseg v0.4.7 h1:WUdvkW8uEhrYfLC4ZzdpI2ztxP1I582+49Oc5Mq64VQ=
+github.com/rivo/uniseg v0.4.7/go.mod h1:FN3SvrM+Zdj16jyLfmOkMNblXMcoc8DfTHruCPUcx88=
github.com/rogpeppe/fastuuid v0.0.0-20150106093220-6724a57986af/go.mod h1:XWv6SoW27p1b0cqNHllgS5HIMJraePCO15w5zCzIWYg=
github.com/rogpeppe/go-internal v1.3.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4=
github.com/rogpeppe/go-internal v1.6.1/go.mod h1:xXDCJY+GAPziupqXw64V24skbSoqbTEfhy4qGm1nDQc=
@@ -1106,11 +1117,11 @@ github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9de
github.com/yuin/goldmark v1.1.32/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74=
github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74=
github.com/yuin/goldmark v1.4.13/go.mod h1:6yULJ656Px+3vBD8DxQVa3kxgyrAnzto9xy5taEt/CY=
-github.com/yuin/goldmark v1.5.2/go.mod h1:6yULJ656Px+3vBD8DxQVa3kxgyrAnzto9xy5taEt/CY=
-github.com/yuin/goldmark v1.5.4 h1:2uY/xC0roWy8IBEGLgB1ywIoEJFGmRrX21YQcvGZzjU=
-github.com/yuin/goldmark v1.5.4/go.mod h1:6yULJ656Px+3vBD8DxQVa3kxgyrAnzto9xy5taEt/CY=
-github.com/yuin/goldmark-emoji v1.0.1 h1:ctuWEyzGBwiucEqxzwe0SOYDXPAucOrE9NQC18Wa1os=
-github.com/yuin/goldmark-emoji v1.0.1/go.mod h1:2w1E6FEWLcDQkoTE+7HU6QF1F6SLlNGjRIBbIZQFqkQ=
+github.com/yuin/goldmark v1.7.1/go.mod h1:uzxRWxtg69N339t3louHJ7+O03ezfj6PlliRlaOzY1E=
+github.com/yuin/goldmark v1.7.4 h1:BDXOHExt+A7gwPCJgPIIq7ENvceR7we7rOS9TNoLZeg=
+github.com/yuin/goldmark v1.7.4/go.mod h1:uzxRWxtg69N339t3louHJ7+O03ezfj6PlliRlaOzY1E=
+github.com/yuin/goldmark-emoji v1.0.3 h1:aLRkLHOuBR2czCY4R8olwMjID+tENfhyFDMCRhbIQY4=
+github.com/yuin/goldmark-emoji v1.0.3/go.mod h1:tTkZEbwu5wkPmgTcitqddVxY9osFZiavD+r4AzQrh1U=
github.com/yusufpapurcu/wmi v1.2.4 h1:zFUKzehAFReQwLys1b/iSMl+JQGSCSjtVqQn9bBrPo0=
github.com/yusufpapurcu/wmi v1.2.4/go.mod h1:SBZ9tNy3G9/m5Oi98Zks0QjeHVDvuK0qfxQmPyzfmi0=
github.com/zcalusic/sysinfo v1.0.2 h1:nwTTo2a+WQ0NXwo0BGRojOJvJ/5XKvQih+2RrtWqfxc=
@@ -1187,8 +1198,8 @@ golang.org/x/crypto v0.7.0/go.mod h1:pYwdfH91IfpZVANVyUOhSIPZaFoJGxTFbZhFTx+dXZU
golang.org/x/crypto v0.14.0/go.mod h1:MVFd36DqK4CsrnJYDkBA3VC4m2GkXAM0PvzMCn4JQf4=
golang.org/x/crypto v0.19.0/go.mod h1:Iy9bg/ha4yyC70EfRS8jz+B6ybOBKMaSxLj6P6oBDfU=
golang.org/x/crypto v0.23.0/go.mod h1:CKFgDieR+mRhux2Lsu27y0fO304Db0wZe70UKqHu0v8=
-golang.org/x/crypto v0.24.0 h1:mnl8DM0o513X8fdIkmyFE/5hTYxbwYOjDS/+rK6qpRI=
-golang.org/x/crypto v0.24.0/go.mod h1:Z1PMYSOR5nyMcyAVAIQSKCDwalqy85Aqn1x3Ws4L5DM=
+golang.org/x/crypto v0.27.0 h1:GXm2NjJrPaiv/h1tb2UH8QfgC/hOf/+z0p6PT8o1w7A=
+golang.org/x/crypto v0.27.0/go.mod h1:1Xngt8kV6Dvbssa53Ziq6Eqn0HqbZi5Z6R0ZpwQzt70=
golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA=
golang.org/x/exp v0.0.0-20190306152737-a1d7652674e8/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA=
golang.org/x/exp v0.0.0-20190510132918-efd6b22b2522/go.mod h1:ZjyILWgesfNpC6sMxTJOJm9Kp84zZh5NQWvqDGG3Qr8=
@@ -1272,7 +1283,6 @@ golang.org/x/net v0.0.0-20211112202133-69e39bad7dc2/go.mod h1:9nx3DQGgdP8bBQD5qx
golang.org/x/net v0.0.0-20220127200216-cd36cc0744dd/go.mod h1:CfG3xpIq0wQ8r1q4Su4UZFWDARRcnwPjda9FqA0JpMk=
golang.org/x/net v0.0.0-20220225172249-27dd8689420f/go.mod h1:CfG3xpIq0wQ8r1q4Su4UZFWDARRcnwPjda9FqA0JpMk=
golang.org/x/net v0.0.0-20220722155237-a158d28d115b/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c=
-golang.org/x/net v0.0.0-20221002022538-bcab6841153b/go.mod h1:YDH+HFinaLZZlnHAfSS6ZXJJ9M9t4Dl22yv3iI2vPwk=
golang.org/x/net v0.5.0/go.mod h1:DivGGAXEgPSlEBzxGzZI+ZLohi+xUj054jfeKui00ws=
golang.org/x/net v0.6.0/go.mod h1:2Tu9+aMcznHK/AK1HMvgo6xiTLG5rD5rZLDS+rp2Bjs=
golang.org/x/net v0.7.0/go.mod h1:2Tu9+aMcznHK/AK1HMvgo6xiTLG5rD5rZLDS+rp2Bjs=
@@ -1282,8 +1292,8 @@ golang.org/x/net v0.10.0/go.mod h1:0qNGK6F8kojg2nk9dLZ2mShWaEBan6FAoqfSigmmuDg=
golang.org/x/net v0.17.0/go.mod h1:NxSsAGuq816PNPmqtQdLE42eU2Fs7NoRIZrHJAlaCOE=
golang.org/x/net v0.21.0/go.mod h1:bIjVDfnllIU7BJ2DNgfnXvpSvtn8VRwhlsaeUTyUS44=
golang.org/x/net v0.25.0/go.mod h1:JkAGAh7GEvH74S6FOH42FLoXpXbE/aqXSrIQjXgsiwM=
-golang.org/x/net v0.26.0 h1:soB7SVo0PWrY4vPW/+ay0jKDNScG2X9wFeYlXIvJsOQ=
-golang.org/x/net v0.26.0/go.mod h1:5YKkiSynbBIh3p6iOc/vibscux0x38BZDkn8sCUPxHE=
+golang.org/x/net v0.29.0 h1:5ORfpBpCs4HzDYoodCDBbwHzdR5UrLBZ3sOnUJmFoHo=
+golang.org/x/net v0.29.0/go.mod h1:gLkgy8jTGERgjzMic6DS9+SP0ajcu6Xu3Orq/SpETg0=
golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U=
golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw=
golang.org/x/oauth2 v0.0.0-20190604053449-0f29369cfe45/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw=
@@ -1308,8 +1318,8 @@ golang.org/x/sync v0.0.0-20201207232520-09787c993a3a/go.mod h1:RxMgew5VJxzue5/jJ
golang.org/x/sync v0.0.0-20220601150217-0de741cfad7f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
golang.org/x/sync v0.1.0/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
-golang.org/x/sync v0.7.0 h1:YsImfSBoP9QPYL0xyKJPq0gcaJdG3rInoqxTWbfQu9M=
-golang.org/x/sync v0.7.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk=
+golang.org/x/sync v0.8.0 h1:3NFvSEYkUoMifnESzZl15y791HH1qU2xm6eCJU5ZPXQ=
+golang.org/x/sync v0.8.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk=
golang.org/x/sys v0.0.0-20180823144017-11551d06cbcc/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
golang.org/x/sys v0.0.0-20180905080454-ebe1bf3edb33/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
@@ -1372,7 +1382,6 @@ golang.org/x/sys v0.0.0-20220330033206-e17cdc41300f/go.mod h1:oPkhp1MJrh7nUepCBc
golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.0.0-20220715151400-c0bba94af5f8/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.0.0-20220722155257-8c9f86f7a55f/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
-golang.org/x/sys v0.0.0-20220728004956-3c1f35247d10/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.0.0-20220811171246-fbc7d0a398ab/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.1.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.3.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
@@ -1385,8 +1394,8 @@ golang.org/x/sys v0.11.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.13.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.17.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
golang.org/x/sys v0.20.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
-golang.org/x/sys v0.21.0 h1:rF+pYz3DAGSQAxAu1CbC7catZg4ebC4UIeIhKxBZvws=
-golang.org/x/sys v0.21.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
+golang.org/x/sys v0.25.0 h1:r+8e+loiHxRqhXVl6ML1nO3l1+oFoWbnlu2Ehimmi34=
+golang.org/x/sys v0.25.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
golang.org/x/term v0.0.0-20201117132131-f5c789dd3221/go.mod h1:Nr5EML6q2oocZ2LXRh80K7BxOlk5/8JxuGnuhpl+muw=
golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo=
golang.org/x/term v0.0.0-20210220032956-6a3ed077a48d/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo=
@@ -1399,8 +1408,8 @@ golang.org/x/term v0.8.0/go.mod h1:xPskH00ivmX89bAKVGSKKtLOWNx2+17Eiy94tnKShWo=
golang.org/x/term v0.13.0/go.mod h1:LTmsnFJwVN6bCy1rVCoS+qHT1HhALEFxKncY3WNNh4U=
golang.org/x/term v0.17.0/go.mod h1:lLRBjIVuehSbZlaOtGMbcMncT+aqLLLmKrsjNrUguwk=
golang.org/x/term v0.20.0/go.mod h1:8UkIAJTvZgivsXaD6/pH6U9ecQzZ45awqEOzuCvwpFY=
-golang.org/x/term v0.21.0 h1:WVXCp+/EBEHOj53Rvu+7KiT/iElMrO8ACK16SMZ3jaA=
-golang.org/x/term v0.21.0/go.mod h1:ooXLefLobQVslOqselCNF4SxFAaoS6KujMbsGzSDmX0=
+golang.org/x/term v0.24.0 h1:Mh5cbb+Zk2hqqXNO7S1iTjEphVL+jb8ZWaqh/g+JWkM=
+golang.org/x/term v0.24.0/go.mod h1:lOBK/LVxemqiMij05LGJ0tzNr8xlmwBRJ81PX6wVLH8=
golang.org/x/text v0.0.0-20170915032832-14c0d48ead0c/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
golang.org/x/text v0.3.1-0.20180807135948-17ff2d5776d2/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
@@ -1417,8 +1426,8 @@ golang.org/x/text v0.9.0/go.mod h1:e1OnstbJyHTd6l/uOt8jFFHp6TRDWZR/bV3emEE/zU8=
golang.org/x/text v0.13.0/go.mod h1:TvPlkZtksWOMsz7fbANvkp4WM8x/WCo/om8BMLbz+aE=
golang.org/x/text v0.14.0/go.mod h1:18ZOQIKpY8NJVqYksKHtTdi31H5itFRjB5/qKTNYzSU=
golang.org/x/text v0.15.0/go.mod h1:18ZOQIKpY8NJVqYksKHtTdi31H5itFRjB5/qKTNYzSU=
-golang.org/x/text v0.16.0 h1:a94ExnEXNtEwYLGJSIUxnWoxoRz/ZcCsV63ROupILh4=
-golang.org/x/text v0.16.0/go.mod h1:GhwF1Be+LQoKShO3cGOHzqOgRrGaYc9AvblQOmPVHnI=
+golang.org/x/text v0.18.0 h1:XvMDiNzPAl0jr17s6W9lcaIhGUfUORdGCNsuLmPG224=
+golang.org/x/text v0.18.0/go.mod h1:BuEKDfySbSR4drPmRPG/7iBdf8hvFMuRexcpahXilzY=
golang.org/x/time v0.0.0-20180412165947-fbb02b2291d2/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
golang.org/x/time v0.0.0-20181108054448-85acf8d2951c/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
golang.org/x/time v0.0.0-20190308202827-9d24e82272b4/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
From c9f67897c483ff9ad436fad0682e2fa1d2c2406a Mon Sep 17 00:00:00 2001
From: Dwi Siswanto
Date: Wed, 25 Sep 2024 23:30:39 +0700
Subject: [PATCH 216/283] fix(http): prevent `addCNameIfAvailable` from using
closed `Dialer` (#5665)
added a check in `addCNameIfAvailable` to ensure
the `Dialer` isnot NIL before attempting to fetch
DNS data.
this prevents potential panics (ex. SIGSEGV) when
the `Dialer` is closed due to an interruption.
Signed-off-by: Dwi Siswanto
---
pkg/protocols/http/request.go | 4 ++++
1 file changed, 4 insertions(+)
diff --git a/pkg/protocols/http/request.go b/pkg/protocols/http/request.go
index 5a1219b83f..180ee512ce 100644
--- a/pkg/protocols/http/request.go
+++ b/pkg/protocols/http/request.go
@@ -1042,6 +1042,10 @@ func (request *Request) validateNFixEvent(input *contextargs.Context, gr *genera
// addCNameIfAvailable adds the cname to the event if available
func (request *Request) addCNameIfAvailable(hostname string, outputEvent map[string]interface{}) {
+ if protocolstate.Dialer == nil {
+ return
+ }
+
data, err := protocolstate.Dialer.GetDNSData(hostname)
if err == nil {
switch len(data.CNAME) {
From e4dae52d5a8828ce4d9c87c4b6880249ffe191c7 Mon Sep 17 00:00:00 2001
From: sandeep <8293321+ehsandeep@users.noreply.github.com>
Date: Thu, 26 Sep 2024 12:09:13 +0400
Subject: [PATCH 217/283] version update
---
pkg/catalog/config/constants.go | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/pkg/catalog/config/constants.go b/pkg/catalog/config/constants.go
index ff54ce9bc0..bc657aaef7 100644
--- a/pkg/catalog/config/constants.go
+++ b/pkg/catalog/config/constants.go
@@ -31,7 +31,7 @@ const (
CLIConfigFileName = "config.yaml"
ReportingConfigFilename = "reporting-config.yaml"
// Version is the current version of nuclei
- Version = `v3.3.2`
+ Version = `v3.3.3`
// Directory Names of custom templates
CustomS3TemplatesDirName = "s3"
CustomGitHubTemplatesDirName = "github"
From 1f945d6d505aed8a6bcced2bd1d76c74c38728a1 Mon Sep 17 00:00:00 2001
From: Tarun Koyalwar <45962551+tarunKoyalwar@users.noreply.github.com>
Date: Sat, 28 Sep 2024 17:20:35 +0400
Subject: [PATCH 218/283] consider protocolType in max host error (#5668)
* consider protocolType in max host error
* add mutex when updating internal-event
---
pkg/core/executors.go | 2 +-
pkg/core/workflow_execute.go | 2 +-
.../common/hosterrorscache/hosterrorscache.go | 20 +++++++++--------
.../hosterrorscache/hosterrorscache_test.go | 22 +++++++++++--------
pkg/protocols/http/request.go | 4 ++--
pkg/protocols/http/request_fuzz.go | 4 ++--
pkg/protocols/network/request.go | 4 ++--
pkg/templates/cluster.go | 4 ++--
pkg/tmplexec/exec.go | 2 ++
pkg/tmplexec/generic/exec.go | 2 +-
10 files changed, 37 insertions(+), 29 deletions(-)
diff --git a/pkg/core/executors.go b/pkg/core/executors.go
index 8447efed13..39ec97aa40 100644
--- a/pkg/core/executors.go
+++ b/pkg/core/executors.go
@@ -107,7 +107,7 @@ func (e *Engine) executeTemplateWithTargets(ctx context.Context, template *templ
currentInfo.Unlock()
// Skip if the host has had errors
- if e.executerOpts.HostErrorsCache != nil && e.executerOpts.HostErrorsCache.Check(contextargs.NewWithMetaInput(ctx, scannedValue)) {
+ if e.executerOpts.HostErrorsCache != nil && e.executerOpts.HostErrorsCache.Check(e.executerOpts.ProtocolType.String(), contextargs.NewWithMetaInput(ctx, scannedValue)) {
return true
}
diff --git a/pkg/core/workflow_execute.go b/pkg/core/workflow_execute.go
index 0b5d7e8722..22b1b813f7 100644
--- a/pkg/core/workflow_execute.go
+++ b/pkg/core/workflow_execute.go
@@ -98,7 +98,7 @@ func (e *Engine) runWorkflowStep(template *workflows.WorkflowTemplate, ctx *scan
}
if err != nil {
if w.Options.HostErrorsCache != nil {
- w.Options.HostErrorsCache.MarkFailed(ctx.Input, err)
+ w.Options.HostErrorsCache.MarkFailed(w.Options.ProtocolType.String(), ctx.Input, err)
}
if len(template.Executers) == 1 {
mainErr = err
diff --git a/pkg/protocols/common/hosterrorscache/hosterrorscache.go b/pkg/protocols/common/hosterrorscache/hosterrorscache.go
index bcfa27dbe4..bca4803e8a 100644
--- a/pkg/protocols/common/hosterrorscache/hosterrorscache.go
+++ b/pkg/protocols/common/hosterrorscache/hosterrorscache.go
@@ -20,10 +20,10 @@ import (
// CacheInterface defines the signature of the hosterrorscache so that
// users of Nuclei as embedded lib may implement their own cache
type CacheInterface interface {
- SetVerbose(verbose bool) // log verbosely
- Close() // close the cache
- Check(ctx *contextargs.Context) bool // return true if the host should be skipped
- MarkFailed(ctx *contextargs.Context, err error) // record a failure (and cause) for the host
+ SetVerbose(verbose bool) // log verbosely
+ Close() // close the cache
+ Check(protoType string, ctx *contextargs.Context) bool // return true if the host should be skipped
+ MarkFailed(protoType string, ctx *contextargs.Context, err error) // record a failure (and cause) for the host
}
var (
@@ -115,7 +115,7 @@ func (c *Cache) NormalizeCacheValue(value string) string {
// - URL: https?:// type
// - Host:port type
// - host type
-func (c *Cache) Check(ctx *contextargs.Context) bool {
+func (c *Cache) Check(protoType string, ctx *contextargs.Context) bool {
finalValue := c.GetKeyFromContext(ctx, nil)
existingCacheItem, err := c.failedTargets.GetIFPresent(finalValue)
@@ -138,8 +138,8 @@ func (c *Cache) Check(ctx *contextargs.Context) bool {
}
// MarkFailed marks a host as failed previously
-func (c *Cache) MarkFailed(ctx *contextargs.Context, err error) {
- if !c.checkError(err) {
+func (c *Cache) MarkFailed(protoType string, ctx *contextargs.Context, err error) {
+ if !c.checkError(protoType, err) {
return
}
finalValue := c.GetKeyFromContext(ctx, err)
@@ -186,11 +186,13 @@ var reCheckError = regexp.MustCompile(`(no address found for host|could not reso
// added to the host skipping table.
// it first parses error and extracts the cause and checks for blacklisted
// or common errors that should be skipped
-func (c *Cache) checkError(err error) bool {
+func (c *Cache) checkError(protoType string, err error) bool {
if err == nil {
return false
}
-
+ if protoType != "http" {
+ return false
+ }
kind := errkit.GetErrorKind(err, nucleierr.ErrTemplateLogic)
switch kind {
case nucleierr.ErrTemplateLogic:
diff --git a/pkg/protocols/common/hosterrorscache/hosterrorscache_test.go b/pkg/protocols/common/hosterrorscache/hosterrorscache_test.go
index 3c93177674..9977b968d9 100644
--- a/pkg/protocols/common/hosterrorscache/hosterrorscache_test.go
+++ b/pkg/protocols/common/hosterrorscache/hosterrorscache_test.go
@@ -11,12 +11,16 @@ import (
"github.com/stretchr/testify/require"
)
+const (
+ protoType = "http"
+)
+
func TestCacheCheck(t *testing.T) {
cache := New(3, DefaultMaxHostsCount, nil)
for i := 0; i < 100; i++ {
- cache.MarkFailed(newCtxArgs("test"), fmt.Errorf("could not resolve host"))
- got := cache.Check(newCtxArgs("test"))
+ cache.MarkFailed(protoType, newCtxArgs("test"), fmt.Errorf("could not resolve host"))
+ got := cache.Check(protoType, newCtxArgs("test"))
if i < 2 {
// till 3 the host is not flagged to skip
require.False(t, got)
@@ -26,7 +30,7 @@ func TestCacheCheck(t *testing.T) {
}
}
- value := cache.Check(newCtxArgs("test"))
+ value := cache.Check(protoType, newCtxArgs("test"))
require.Equal(t, true, value, "could not get checked value")
}
@@ -34,8 +38,8 @@ func TestTrackErrors(t *testing.T) {
cache := New(3, DefaultMaxHostsCount, []string{"custom error"})
for i := 0; i < 100; i++ {
- cache.MarkFailed(newCtxArgs("custom"), fmt.Errorf("got: nested: custom error"))
- got := cache.Check(newCtxArgs("custom"))
+ cache.MarkFailed(protoType, newCtxArgs("custom"), fmt.Errorf("got: nested: custom error"))
+ got := cache.Check(protoType, newCtxArgs("custom"))
if i < 2 {
// till 3 the host is not flagged to skip
require.False(t, got)
@@ -44,7 +48,7 @@ func TestTrackErrors(t *testing.T) {
require.True(t, got)
}
}
- value := cache.Check(newCtxArgs("custom"))
+ value := cache.Check(protoType, newCtxArgs("custom"))
require.Equal(t, true, value, "could not get checked value")
}
@@ -86,7 +90,7 @@ func TestCacheMarkFailed(t *testing.T) {
for _, test := range tests {
normalizedCacheValue := cache.GetKeyFromContext(newCtxArgs(test.host), nil)
- cache.MarkFailed(newCtxArgs(test.host), fmt.Errorf("no address found for host"))
+ cache.MarkFailed(protoType, newCtxArgs(test.host), fmt.Errorf("no address found for host"))
failedTarget, err := cache.failedTargets.Get(normalizedCacheValue)
require.Nil(t, err)
require.NotNil(t, failedTarget)
@@ -122,14 +126,14 @@ func TestCacheMarkFailedConcurrent(t *testing.T) {
wg.Add(1)
go func() {
defer wg.Done()
- cache.MarkFailed(newCtxArgs(currentTest.host), fmt.Errorf("could not resolve host"))
+ cache.MarkFailed(protoType, newCtxArgs(currentTest.host), fmt.Errorf("could not resolve host"))
}()
}
}
wg.Wait()
for _, test := range tests {
- require.True(t, cache.Check(newCtxArgs(test.host)))
+ require.True(t, cache.Check(protoType, newCtxArgs(test.host)))
normalizedCacheValue := cache.NormalizeCacheValue(test.host)
failedTarget, err := cache.failedTargets.Get(normalizedCacheValue)
diff --git a/pkg/protocols/http/request.go b/pkg/protocols/http/request.go
index 180ee512ce..994e065582 100644
--- a/pkg/protocols/http/request.go
+++ b/pkg/protocols/http/request.go
@@ -1177,14 +1177,14 @@ func (request *Request) markUnresponsiveAddress(input *contextargs.Context, err
return
}
if request.options.HostErrorsCache != nil {
- request.options.HostErrorsCache.MarkFailed(input, err)
+ request.options.HostErrorsCache.MarkFailed(request.options.ProtocolType.String(), input, err)
}
}
// isUnresponsiveAddress checks if the error is a unreponsive based on its execution history
func (request *Request) isUnresponsiveAddress(input *contextargs.Context) bool {
if request.options.HostErrorsCache != nil {
- return request.options.HostErrorsCache.Check(input)
+ return request.options.HostErrorsCache.Check(request.options.ProtocolType.String(), input)
}
return false
}
diff --git a/pkg/protocols/http/request_fuzz.go b/pkg/protocols/http/request_fuzz.go
index fdf862eb67..a7c6e80c07 100644
--- a/pkg/protocols/http/request_fuzz.go
+++ b/pkg/protocols/http/request_fuzz.go
@@ -161,7 +161,7 @@ func (request *Request) executeAllFuzzingRules(input *contextargs.Context, value
func (request *Request) executeGeneratedFuzzingRequest(gr fuzz.GeneratedRequest, input *contextargs.Context, callback protocols.OutputEventCallback) bool {
hasInteractMatchers := interactsh.HasMatchers(request.CompiledOperators)
hasInteractMarkers := len(gr.InteractURLs) > 0
- if request.options.HostErrorsCache != nil && request.options.HostErrorsCache.Check(input) {
+ if request.options.HostErrorsCache != nil && request.options.HostErrorsCache.Check(request.options.ProtocolType.String(), input) {
return false
}
request.options.RateLimitTake()
@@ -215,7 +215,7 @@ func (request *Request) executeGeneratedFuzzingRequest(gr fuzz.GeneratedRequest,
}
if requestErr != nil {
if request.options.HostErrorsCache != nil {
- request.options.HostErrorsCache.MarkFailed(input, requestErr)
+ request.options.HostErrorsCache.MarkFailed(request.options.ProtocolType.String(), input, requestErr)
}
gologger.Verbose().Msgf("[%s] Error occurred in request: %s\n", request.options.TemplateID, requestErr)
}
diff --git a/pkg/protocols/network/request.go b/pkg/protocols/network/request.go
index 5fa8609d51..32d4ae3494 100644
--- a/pkg/protocols/network/request.go
+++ b/pkg/protocols/network/request.go
@@ -504,14 +504,14 @@ func (request *Request) markUnresponsiveAddress(input *contextargs.Context, err
return
}
if request.options.HostErrorsCache != nil {
- request.options.HostErrorsCache.MarkFailed(input, err)
+ request.options.HostErrorsCache.MarkFailed(request.options.ProtocolType.String(), input, err)
}
}
// isUnresponsiveAddress checks if the error is a unreponsive based on its execution history
func (request *Request) isUnresponsiveAddress(input *contextargs.Context) bool {
if request.options.HostErrorsCache != nil {
- return request.options.HostErrorsCache.Check(input)
+ return request.options.HostErrorsCache.Check(request.options.ProtocolType.String(), input)
}
return false
}
diff --git a/pkg/templates/cluster.go b/pkg/templates/cluster.go
index 8f1af96b74..63b065d346 100644
--- a/pkg/templates/cluster.go
+++ b/pkg/templates/cluster.go
@@ -274,7 +274,7 @@ func (e *ClusterExecuter) Execute(ctx *scan.ScanContext) (bool, error) {
}
})
if err != nil && e.options.HostErrorsCache != nil {
- e.options.HostErrorsCache.MarkFailed(ctx.Input, err)
+ e.options.HostErrorsCache.MarkFailed(e.options.ProtocolType.String(), ctx.Input, err)
}
return results, err
}
@@ -310,7 +310,7 @@ func (e *ClusterExecuter) ExecuteWithResults(ctx *scan.ScanContext) ([]*output.R
}
if err != nil && e.options.HostErrorsCache != nil {
- e.options.HostErrorsCache.MarkFailed(ctx.Input, err)
+ e.options.HostErrorsCache.MarkFailed(e.options.ProtocolType.String(), ctx.Input, err)
}
return scanCtx.GenerateResult(), err
}
diff --git a/pkg/tmplexec/exec.go b/pkg/tmplexec/exec.go
index 149deaa4d5..279d03d849 100644
--- a/pkg/tmplexec/exec.go
+++ b/pkg/tmplexec/exec.go
@@ -206,7 +206,9 @@ func (e *TemplateExecuter) Execute(ctx *scan.ScanContext) (bool, error) {
ctx.LogError(errx)
if lastMatcherEvent != nil {
+ lastMatcherEvent.Lock()
lastMatcherEvent.InternalEvent["error"] = getErrorCause(ctx.GenerateErrorMessage())
+ lastMatcherEvent.Unlock()
writeFailureCallback(lastMatcherEvent, e.options.Options.MatcherStatus)
}
diff --git a/pkg/tmplexec/generic/exec.go b/pkg/tmplexec/generic/exec.go
index 3b337f6b20..c8303f70d9 100644
--- a/pkg/tmplexec/generic/exec.go
+++ b/pkg/tmplexec/generic/exec.go
@@ -85,7 +85,7 @@ func (g *Generic) ExecuteWithResults(ctx *scan.ScanContext) error {
if err != nil {
ctx.LogError(err)
if g.options.HostErrorsCache != nil {
- g.options.HostErrorsCache.MarkFailed(ctx.Input, err)
+ g.options.HostErrorsCache.MarkFailed(g.options.ProtocolType.String(), ctx.Input, err)
}
gologger.Warning().Msgf("[%s] Could not execute request for %s: %s\n", g.options.TemplateID, ctx.Input.MetaInput.PrettyPrint(), err)
}
From d1614857bd345affb24f209aeebde94db7303b6d Mon Sep 17 00:00:00 2001
From: sandeep <8293321+ehsandeep@users.noreply.github.com>
Date: Sat, 28 Sep 2024 19:06:40 +0400
Subject: [PATCH 219/283] version update
---
pkg/catalog/config/constants.go | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/pkg/catalog/config/constants.go b/pkg/catalog/config/constants.go
index bc657aaef7..e29af00cdd 100644
--- a/pkg/catalog/config/constants.go
+++ b/pkg/catalog/config/constants.go
@@ -31,7 +31,7 @@ const (
CLIConfigFileName = "config.yaml"
ReportingConfigFilename = "reporting-config.yaml"
// Version is the current version of nuclei
- Version = `v3.3.3`
+ Version = `v3.3.4`
// Directory Names of custom templates
CustomS3TemplatesDirName = "s3"
CustomGitHubTemplatesDirName = "github"
From d0463e8ca39cf28e5e25b6426d041eab4c30b569 Mon Sep 17 00:00:00 2001
From: "alban.stourbe stourbe"
Date: Mon, 30 Sep 2024 16:52:54 +0200
Subject: [PATCH 220/283] Add vars with SDK
---
lib/config.go | 14 ++++++++++++++
1 file changed, 14 insertions(+)
diff --git a/lib/config.go b/lib/config.go
index 7a9419aa50..e2a56c1375 100644
--- a/lib/config.go
+++ b/lib/config.go
@@ -392,6 +392,20 @@ func WithHeaders(headers []string) NucleiSDKOptions {
}
}
+// WithVars allows setting custom variables to use in templates/workflows context
+func WithVars(vars []string) NucleiSDKOptions {
+ // Create a goflags.RuntimeMap
+ runtimeVars := goflags.RuntimeMap{}
+ for _, v := range vars {
+ runtimeVars.Set(v)
+ }
+
+ return func(e *NucleiEngine) error {
+ e.opts.Vars = runtimeVars
+ return nil
+ }
+}
+
// EnablePassiveMode allows enabling passive HTTP response processing mode
func EnablePassiveMode() NucleiSDKOptions {
return func(e *NucleiEngine) error {
From ed3e06a7011a92da046549916a2e499b3a741e4b Mon Sep 17 00:00:00 2001
From: "alban.stourbe stourbe"
Date: Tue, 1 Oct 2024 09:29:43 +0200
Subject: [PATCH 221/283] Add checkerr on runtimeVars & Add a new TestSDK
---
lib/config.go | 7 ++++++-
lib/tests/sdk_test.go | 33 +++++++++++++++++++++++++++++++++
2 files changed, 39 insertions(+), 1 deletion(-)
diff --git a/lib/config.go b/lib/config.go
index e2a56c1375..20b3449c4c 100644
--- a/lib/config.go
+++ b/lib/config.go
@@ -397,7 +397,12 @@ func WithVars(vars []string) NucleiSDKOptions {
// Create a goflags.RuntimeMap
runtimeVars := goflags.RuntimeMap{}
for _, v := range vars {
- runtimeVars.Set(v)
+ err := runtimeVars.Set(v)
+ if err != nil {
+ return func(e *NucleiEngine) error {
+ return err
+ }
+ }
}
return func(e *NucleiEngine) error {
diff --git a/lib/tests/sdk_test.go b/lib/tests/sdk_test.go
index 97ec489abc..395acfd515 100644
--- a/lib/tests/sdk_test.go
+++ b/lib/tests/sdk_test.go
@@ -133,3 +133,36 @@ func TestThreadSafeNuclei(t *testing.T) {
fn()
}
}
+
+func TestWithVarsNuclei(t *testing.T) {
+ fn := func() {
+ defer func() {
+ // resources like leveldb have a delay to commit in-memory resources
+ // to disk, typically 1-2 seconds, so we wait for 2 seconds
+ time.Sleep(2 * time.Second)
+ goleak.VerifyNone(t, knownLeaks...)
+ }()
+ ne, err := nuclei.NewNucleiEngineCtx(
+ context.TODO(),
+ nuclei.WithTemplatesOrWorkflows(nuclei.TemplateSources{Templates: []string{"http/token-spray/api-1forge.yaml"}}),
+ nuclei.WithVars([]string{"token=foobar"}),
+ nuclei.WithVerbosity(nuclei.VerbosityOptions{Debug: true}),
+ )
+ require.Nil(t, err)
+ ne.LoadTargets([]string{"scanme.sh"}, true) // probe http/https target is set to true here
+ err = ne.ExecuteWithCallback(nil)
+ require.Nil(t, err)
+ defer ne.Close()
+ }
+ // this is shared test so needs to be run as seperate process
+ if env.GetEnvOrDefault("TestWithVarsNuclei", false) {
+ cmd := exec.Command(os.Args[0], "-test.run=TestWithVarsNuclei")
+ cmd.Env = append(os.Environ(), "TestWithVarsNuclei=true")
+ out, err := cmd.CombinedOutput()
+ if err != nil {
+ t.Fatalf("process ran with error %s, output: %s", err, out)
+ }
+ } else {
+ fn()
+ }
+}
From 23825c77b2869dd3dce9f15f4d4b4bde3ec2c63a Mon Sep 17 00:00:00 2001
From: Piotr Idzik <65706193+vil02@users.noreply.github.com>
Date: Fri, 4 Oct 2024 23:07:07 +0200
Subject: [PATCH 222/283] style: do not use backticks (#5687)
---
gh_retry.sh | 8 ++++----
1 file changed, 4 insertions(+), 4 deletions(-)
diff --git a/gh_retry.sh b/gh_retry.sh
index bf212e1290..43542276b3 100755
--- a/gh_retry.sh
+++ b/gh_retry.sh
@@ -21,9 +21,9 @@ WORKFLOW="Build Test"
GREP_ERROR_PATTERN='Test "http/interactsh.yaml" failed'
#Set fonts for Help.
-NORM=`tput sgr0`
-BOLD=`tput bold`
-REV=`tput smso`
+NORM=$(tput sgr0)
+BOLD=$(tput bold)
+REV=$(tput smso)
HELP()
{
@@ -73,7 +73,7 @@ function print_bold() {
function retry_failed_jobs() {
print_bold "Checking failed workflows for branch $BRANCH before $BEFORE"
- date=`date +%Y-%m-%d'T'%H:%M'Z' -d "$BEFORE"`
+ date=$(date +%Y-%m-%d'T'%H:%M'Z' -d "$BEFORE")
workflowIds=$(gh run list --limit "$LIMIT" --json headBranch,status,name,conclusion,databaseId,updatedAt | jq -c '.[] |
select ( .headBranch==$branch ) |
From 8b9acb292780cb8647db27b517cf5d2c0fa782f2 Mon Sep 17 00:00:00 2001
From: Ramana Reddy <90540245+RamanaReddy0M@users.noreply.github.com>
Date: Mon, 7 Oct 2024 18:11:03 +0530
Subject: [PATCH 223/283] return bool resp on successful ldap authentication
(#5682)
---
pkg/js/libs/ldap/ldap.go | 16 ++++++++++------
1 file changed, 10 insertions(+), 6 deletions(-)
diff --git a/pkg/js/libs/ldap/ldap.go b/pkg/js/libs/ldap/ldap.go
index 3961fa7b7e..e10015acfa 100644
--- a/pkg/js/libs/ldap/ldap.go
+++ b/pkg/js/libs/ldap/ldap.go
@@ -155,7 +155,7 @@ func NewClient(call goja.ConstructorCall, runtime *goja.Runtime) *goja.Object {
// const client = new ldap.Client('ldap://ldap.example.com', 'acme.com');
// client.Authenticate('user', 'password');
// ```
-func (c *Client) Authenticate(username, password string) {
+func (c *Client) Authenticate(username, password string) bool {
c.nj.Require(c.conn != nil, "no existing connection")
if c.BaseDN == "" {
c.BaseDN = fmt.Sprintf("dc=%s", strings.Join(strings.Split(c.Realm, "."), ",dc="))
@@ -163,19 +163,21 @@ func (c *Client) Authenticate(username, password string) {
if err := c.conn.NTLMBind(c.Realm, username, password); err == nil {
// if bind with NTLMBind(), there is nothing
// else to do, you are authenticated
- return
+ return true
}
+ var err error
switch password {
case "":
- if err := c.conn.UnauthenticatedBind(username); err != nil {
+ if err = c.conn.UnauthenticatedBind(username); err != nil {
c.nj.ThrowError(err)
}
default:
- if err := c.conn.Bind(username, password); err != nil {
+ if err = c.conn.Bind(username, password); err != nil {
c.nj.ThrowError(err)
}
}
+ return err == nil
}
// AuthenticateWithNTLMHash authenticates with the ldap server using the given username and NTLM hash
@@ -185,14 +187,16 @@ func (c *Client) Authenticate(username, password string) {
// const client = new ldap.Client('ldap://ldap.example.com', 'acme.com');
// client.AuthenticateWithNTLMHash('pdtm', 'hash');
// ```
-func (c *Client) AuthenticateWithNTLMHash(username, hash string) {
+func (c *Client) AuthenticateWithNTLMHash(username, hash string) bool {
c.nj.Require(c.conn != nil, "no existing connection")
if c.BaseDN == "" {
c.BaseDN = fmt.Sprintf("dc=%s", strings.Join(strings.Split(c.Realm, "."), ",dc="))
}
- if err := c.conn.NTLMBindWithHash(c.Realm, username, hash); err != nil {
+ var err error
+ if err = c.conn.NTLMBindWithHash(c.Realm, username, hash); err != nil {
c.nj.ThrowError(err)
}
+ return err == nil
}
// Search accepts whatever filter and returns a list of maps having provided attributes
From 7ba5d51b0002410d30f163c7fdd866bf23e7dc5c Mon Sep 17 00:00:00 2001
From: Ramana Reddy <90540245+RamanaReddy0M@users.noreply.github.com>
Date: Mon, 7 Oct 2024 18:12:07 +0530
Subject: [PATCH 224/283] fix: ldap metadata collection err (#5683)
---
pkg/js/libs/ldap/ldap.go | 3 ++-
1 file changed, 2 insertions(+), 1 deletion(-)
diff --git a/pkg/js/libs/ldap/ldap.go b/pkg/js/libs/ldap/ldap.go
index e10015acfa..a8f0eb0c8e 100644
--- a/pkg/js/libs/ldap/ldap.go
+++ b/pkg/js/libs/ldap/ldap.go
@@ -283,9 +283,10 @@ func (c *Client) CollectMetadata() Metadata {
}
metadata.BaseDN = c.BaseDN
+ // Use scope as Base since Root DSE doesn't have subentries
srMetadata := ldap.NewSearchRequest(
"",
- ldap.ScopeWholeSubtree,
+ ldap.ScopeBaseObject,
ldap.NeverDerefAliases,
0, 0, false,
"(objectClass=*)",
From f0624820d314d70b86742765e6a335ebb54c6dda Mon Sep 17 00:00:00 2001
From: Dogan Can Bakir <65292895+dogancanbakir@users.noreply.github.com>
Date: Thu, 10 Oct 2024 00:34:23 +0300
Subject: [PATCH 225/283] update ssl part definitions (#5710)
---
pkg/protocols/ssl/ssl.go | 32 +++++++++++++++++++++++++++-----
1 file changed, 27 insertions(+), 5 deletions(-)
diff --git a/pkg/protocols/ssl/ssl.go b/pkg/protocols/ssl/ssl.go
index 681043d3bf..554a4f5c10 100644
--- a/pkg/protocols/ssl/ssl.go
+++ b/pkg/protocols/ssl/ssl.go
@@ -347,11 +347,33 @@ func (request *Request) ExecuteWithResults(input *contextargs.Context, dynamicVa
// description. Multiple definitions are separated by commas.
// Definitions not having a name (generated on runtime) are prefixed & suffixed by <>.
var RequestPartDefinitions = map[string]string{
- "type": "Type is the type of request made",
- "response": "JSON SSL protocol handshake details",
- "not_after": "Timestamp after which the remote cert expires",
- "host": "Host is the input to the template",
- "matched": "Matched is the input which was matched upon",
+ "template-id": "ID of the template executed",
+ "template-info": "Info Block of the template executed",
+ "template-path": "Path of the template executed",
+ "host": "Host is the input to the template",
+ "port": "Port is the port of the host",
+ "matched": "Matched is the input which was matched upon",
+ "type": "Type is the type of request made",
+ "timestamp": "Timestamp is the time when the request was made",
+ "response": "JSON SSL protocol handshake details",
+ "cipher": "Cipher is the encryption algorithm used",
+ "domains": "Domains are the list of domain names in the certificate",
+ "fingerprint_hash": "Fingerprint hash is the unique identifier of the certificate",
+ "ip": "IP is the IP address of the server",
+ "issuer_cn": "Issuer CN is the common name of the certificate issuer",
+ "issuer_dn": "Issuer DN is the distinguished name of the certificate issuer",
+ "issuer_org": "Issuer organization is the organization of the certificate issuer",
+ "not_after": "Timestamp after which the remote cert expires",
+ "not_before": "Timestamp before which the certificate is not valid",
+ "probe_status": "Probe status indicates if the probe was successful",
+ "serial": "Serial is the serial number of the certificate",
+ "sni": "SNI is the server name indication used in the handshake",
+ "subject_an": "Subject AN is the list of subject alternative names",
+ "subject_cn": "Subject CN is the common name of the certificate subject",
+ "subject_dn": "Subject DN is the distinguished name of the certificate subject",
+ "subject_org": "Subject organization is the organization of the certificate subject",
+ "tls_connection": "TLS connection is the type of TLS connection used",
+ "tls_version": "TLS version is the version of the TLS protocol used",
}
// Match performs matching operation for a matcher on model and returns:
From 690089e1ce723cc84ff89a0c1cd1ea7ba64f6d15 Mon Sep 17 00:00:00 2001
From: ghost
Date: Wed, 9 Oct 2024 21:36:38 +0000
Subject: [PATCH 226/283] Auto Generate Syntax Docs + JSONSchema [Wed Oct 9
21:36:38 UTC 2024] :robot:
---
SYNTAX-REFERENCE.md | 26 ++++++++-
pkg/templates/templates_doc.go | 96 ++++++++++++++++++++++++++++++++--
2 files changed, 116 insertions(+), 6 deletions(-)
diff --git a/SYNTAX-REFERENCE.md b/SYNTAX-REFERENCE.md
index 9ec413d3ed..394f0a0684 100755
--- a/SYNTAX-REFERENCE.md
+++ b/SYNTAX-REFERENCE.md
@@ -3764,11 +3764,33 @@ Appears in:
Part Definitions:
+- template-id
- ID of the template executed
+- template-info
- Info Block of the template executed
+- template-path
- Path of the template executed
+- host
- Host is the input to the template
+- port
- Port is the port of the host
+- matched
- Matched is the input which was matched upon
- type
- Type is the type of request made
+- timestamp
- Timestamp is the time when the request was made
- response
- JSON SSL protocol handshake details
+- cipher
- Cipher is the encryption algorithm used
+- domains
- Domains are the list of domain names in the certificate
+- fingerprint_hash
- Fingerprint hash is the unique identifier of the certificate
+- ip
- IP is the IP address of the server
+- issuer_cn
- Issuer CN is the common name of the certificate issuer
+- issuer_dn
- Issuer DN is the distinguished name of the certificate issuer
+- issuer_org
- Issuer organization is the organization of the certificate issuer
- not_after
- Timestamp after which the remote cert expires
-- host
- Host is the input to the template
-- matched
- Matched is the input which was matched upon
+- not_before
- Timestamp before which the certificate is not valid
+- probe_status
- Probe status indicates if the probe was successful
+- serial
- Serial is the serial number of the certificate
+- sni
- SNI is the server name indication used in the handshake
+- subject_an
- Subject AN is the list of subject alternative names
+- subject_cn
- Subject CN is the common name of the certificate subject
+- subject_dn
- Subject DN is the distinguished name of the certificate subject
+- subject_org
- Subject organization is the organization of the certificate subject
+- tls_connection
- TLS connection is the type of TLS connection used
+- tls_version
- TLS version is the version of the TLS protocol used
diff --git a/pkg/templates/templates_doc.go b/pkg/templates/templates_doc.go
index 827e583d3b..b1422c8333 100644
--- a/pkg/templates/templates_doc.go
+++ b/pkg/templates/templates_doc.go
@@ -1656,25 +1656,113 @@ func init() {
},
}
SSLRequestDoc.PartDefinitions = []encoder.KeyValue{
+ {
+ Key: "template-id",
+ Value: "ID of the template executed",
+ },
+ {
+ Key: "template-info",
+ Value: "Info Block of the template executed",
+ },
+ {
+ Key: "template-path",
+ Value: "Path of the template executed",
+ },
+ {
+ Key: "host",
+ Value: "Host is the input to the template",
+ },
+ {
+ Key: "port",
+ Value: "Port is the port of the host",
+ },
+ {
+ Key: "matched",
+ Value: "Matched is the input which was matched upon",
+ },
{
Key: "type",
Value: "Type is the type of request made",
},
+ {
+ Key: "timestamp",
+ Value: "Timestamp is the time when the request was made",
+ },
{
Key: "response",
Value: "JSON SSL protocol handshake details",
},
+ {
+ Key: "cipher",
+ Value: "Cipher is the encryption algorithm used",
+ },
+ {
+ Key: "domains",
+ Value: "Domains are the list of domain names in the certificate",
+ },
+ {
+ Key: "fingerprint_hash",
+ Value: "Fingerprint hash is the unique identifier of the certificate",
+ },
+ {
+ Key: "ip",
+ Value: "IP is the IP address of the server",
+ },
+ {
+ Key: "issuer_cn",
+ Value: "Issuer CN is the common name of the certificate issuer",
+ },
+ {
+ Key: "issuer_dn",
+ Value: "Issuer DN is the distinguished name of the certificate issuer",
+ },
+ {
+ Key: "issuer_org",
+ Value: "Issuer organization is the organization of the certificate issuer",
+ },
{
Key: "not_after",
Value: "Timestamp after which the remote cert expires",
},
{
- Key: "host",
- Value: "Host is the input to the template",
+ Key: "not_before",
+ Value: "Timestamp before which the certificate is not valid",
},
{
- Key: "matched",
- Value: "Matched is the input which was matched upon",
+ Key: "probe_status",
+ Value: "Probe status indicates if the probe was successful",
+ },
+ {
+ Key: "serial",
+ Value: "Serial is the serial number of the certificate",
+ },
+ {
+ Key: "sni",
+ Value: "SNI is the server name indication used in the handshake",
+ },
+ {
+ Key: "subject_an",
+ Value: "Subject AN is the list of subject alternative names",
+ },
+ {
+ Key: "subject_cn",
+ Value: "Subject CN is the common name of the certificate subject",
+ },
+ {
+ Key: "subject_dn",
+ Value: "Subject DN is the distinguished name of the certificate subject",
+ },
+ {
+ Key: "subject_org",
+ Value: "Subject organization is the organization of the certificate subject",
+ },
+ {
+ Key: "tls_connection",
+ Value: "TLS connection is the type of TLS connection used",
+ },
+ {
+ Key: "tls_version",
+ Value: "TLS version is the version of the TLS protocol used",
},
}
SSLRequestDoc.Fields = make([]encoder.Doc, 9)
From 82680980a5a2504eb0fa668de82338b5c13fba09 Mon Sep 17 00:00:00 2001
From: Ice3man
Date: Thu, 10 Oct 2024 20:22:22 +0530
Subject: [PATCH 227/283] bugfix: fix multipart panic + support for filename +
content-type (#5702)
* bugfix: fix multipart files panic + support for filename + content-type propagation
* misc changes
---
pkg/fuzz/dataformat/multipart.go | 44 +++++++++++++++++++++++++++++++-
1 file changed, 43 insertions(+), 1 deletion(-)
diff --git a/pkg/fuzz/dataformat/multipart.go b/pkg/fuzz/dataformat/multipart.go
index d7e40af10c..227025d22b 100644
--- a/pkg/fuzz/dataformat/multipart.go
+++ b/pkg/fuzz/dataformat/multipart.go
@@ -6,12 +6,19 @@ import (
"io"
"mime"
"mime/multipart"
+ "net/textproto"
mapsutil "github.com/projectdiscovery/utils/maps"
)
type MultiPartForm struct {
- boundary string
+ boundary string
+ filesMetadata map[string]FileMetadata
+}
+
+type FileMetadata struct {
+ ContentType string
+ Filename string
}
var (
@@ -41,11 +48,40 @@ func (m *MultiPartForm) Encode(data KV) (string, error) {
data.Iterate(func(key string, value any) bool {
var fw io.Writer
var err error
+
+ if filesArray, ok := value.([]interface{}); ok {
+ fileMetadata, ok := m.filesMetadata[key]
+ if !ok {
+ Itererr = fmt.Errorf("file metadata not found for key %s", key)
+ return false
+ }
+
+ for _, file := range filesArray {
+ h := make(textproto.MIMEHeader)
+ h.Set("Content-Disposition",
+ fmt.Sprintf(`form-data; name=%q; filename=%q`,
+ key, fileMetadata.Filename))
+ h.Set("Content-Type", fileMetadata.ContentType)
+
+ if fw, err = w.CreatePart(h); err != nil {
+ Itererr = err
+ return false
+ }
+
+ if _, err = fw.Write([]byte(file.(string))); err != nil {
+ Itererr = err
+ return false
+ }
+ }
+ return true
+ }
+
// Add field
if fw, err = w.CreateFormField(key); err != nil {
Itererr = err
return false
}
+
if _, err = fw.Write([]byte(value.(string))); err != nil {
Itererr = err
return false
@@ -98,6 +134,7 @@ func (m *MultiPartForm) Decode(data string) (KV, error) {
result.Set(key, values[0])
}
}
+ m.filesMetadata = make(map[string]FileMetadata)
for key, files := range form.File {
fileContents := []interface{}{}
for _, fileHeader := range files {
@@ -112,6 +149,11 @@ func (m *MultiPartForm) Decode(data string) (KV, error) {
return KV{}, err
}
fileContents = append(fileContents, buffer.String())
+
+ m.filesMetadata[key] = FileMetadata{
+ ContentType: fileHeader.Header.Get("Content-Type"),
+ Filename: fileHeader.Filename,
+ }
}
result.Set(key, fileContents)
}
From 1cd42c46c7b4b33e63774983bfea59a543c2b826 Mon Sep 17 00:00:00 2001
From: Dwi Siswanto
Date: Fri, 11 Oct 2024 20:43:02 +0700
Subject: [PATCH 228/283] chore: update `auto_assign` (#5720)
add me to `addReviewers` list
Signed-off-by: Dwi Siswanto
---
.github/auto_assign.yml | 1 +
1 file changed, 1 insertion(+)
diff --git a/.github/auto_assign.yml b/.github/auto_assign.yml
index 0c65e536ca..3d6642fc59 100644
--- a/.github/auto_assign.yml
+++ b/.github/auto_assign.yml
@@ -1,6 +1,7 @@
addReviewers: true
reviewers:
- dogancanbakir
+ - dwisiswant0
numberOfReviewers: 1
skipKeywords:
From 3f0de96726b0dac0304132733e17a1e2d56e8a9f Mon Sep 17 00:00:00 2001
From: Keith Chason
Date: Sun, 13 Oct 2024 12:14:33 -0400
Subject: [PATCH 229/283] MongoDB Reporting (#5688)
* Initial setup of Mongo reporting
* Fix slice pop logic
* Switch to config-file logic
* Parse database name from connection string
* Switch to url.Parse for connection string parsing
* Address return/logging feedback
---
cmd/nuclei/issue-tracker-config.yaml | 11 ++
go.mod | 6 +
go.sum | 12 ++
pkg/reporting/exporters/mongo/mongo.go | 155 +++++++++++++++++++++++++
pkg/reporting/options.go | 3 +
pkg/reporting/reporting.go | 9 ++
6 files changed, 196 insertions(+)
create mode 100644 pkg/reporting/exporters/mongo/mongo.go
diff --git a/cmd/nuclei/issue-tracker-config.yaml b/cmd/nuclei/issue-tracker-config.yaml
index b7e0e6dafc..07ccf828f2 100644
--- a/cmd/nuclei/issue-tracker-config.yaml
+++ b/cmd/nuclei/issue-tracker-config.yaml
@@ -162,3 +162,14 @@
# duplicate-issue-check: false
# # open-state-id is the ID of the open state in Linear
# open-state-id: ""
+#mongodb:
+# # the connection string to the MongoDB database
+# # (e.g., mongodb://root:example@localhost:27017/nuclei?ssl=false&authSource=admin)
+# connection-string: ""
+# # the name of the collection to store the issues
+# collection-name: ""
+# # excludes the Request and Response from the results (helps with filesize)
+# omit-raw: false
+# # determines the number of results to be kept in memory before writing it to the database or 0 to
+# # persist all in memory and write all results at the end (default)
+# batch-size: 0
\ No newline at end of file
diff --git a/go.mod b/go.mod
index f2adacca21..86a8a00643 100644
--- a/go.mod
+++ b/go.mod
@@ -105,6 +105,7 @@ require (
github.com/stretchr/testify v1.9.0
github.com/tarunKoyalwar/goleak v0.0.0-20240429141123-0efa90dbdcf9
github.com/zmap/zgrab2 v0.1.8-0.20230806160807-97ba87c0e706
+ go.mongodb.org/mongo-driver v1.17.0
golang.org/x/term v0.24.0
gopkg.in/yaml.v3 v3.0.1
moul.io/http2curl v1.0.0
@@ -195,6 +196,7 @@ require (
github.com/mitchellh/mapstructure v1.5.0 // indirect
github.com/moby/term v0.5.0 // indirect
github.com/mohae/deepcopy v0.0.0-20170929034955-c48cc78d4826 // indirect
+ github.com/montanaflynn/stats v0.7.1 // indirect
github.com/muesli/reflow v0.3.0 // indirect
github.com/muesli/termenv v0.15.3-0.20240618155329-98d742f6907a // indirect
github.com/opencontainers/go-digest v1.0.0 // indirect
@@ -228,9 +230,13 @@ require (
github.com/twitchyliquid64/golang-asm v0.15.1 // indirect
github.com/ugorji/go/codec v1.2.11 // indirect
github.com/wk8/go-ordered-map/v2 v2.1.8 // indirect
+ github.com/xdg-go/pbkdf2 v1.0.0 // indirect
+ github.com/xdg-go/scram v1.1.2 // indirect
+ github.com/xdg-go/stringprep v1.0.4 // indirect
github.com/xeipuuv/gojsonpointer v0.0.0-20190905194746-02993c407bfb // indirect
github.com/xeipuuv/gojsonreference v0.0.0-20180127040603-bd5ef7bd5415 // indirect
github.com/xeipuuv/gojsonschema v1.2.0 // indirect
+ github.com/youmark/pkcs8 v0.0.0-20240726163527-a2c0da244d78 // indirect
github.com/ysmood/fetchup v0.2.3 // indirect
github.com/ysmood/got v0.34.1 // indirect
github.com/yuin/goldmark v1.7.4 // indirect
diff --git a/go.sum b/go.sum
index da71e9b287..b69d6283a5 100644
--- a/go.sum
+++ b/go.sum
@@ -755,6 +755,8 @@ github.com/modern-go/reflect2 v1.0.2 h1:xBagoLtFs94CBntxluKeaWgTMpvLxC4ur3nMaC9G
github.com/modern-go/reflect2 v1.0.2/go.mod h1:yWuevngMOJpCy52FWWMvUC8ws7m/LJsjYzDa0/r8luk=
github.com/mohae/deepcopy v0.0.0-20170929034955-c48cc78d4826 h1:RWengNIwukTxcDr9M+97sNutRR1RKhG96O6jWumTTnw=
github.com/mohae/deepcopy v0.0.0-20170929034955-c48cc78d4826/go.mod h1:TaXosZuwdSHYgviHp1DAtfrULt5eUgsSMsZf+YrPgl8=
+github.com/montanaflynn/stats v0.7.1 h1:etflOAAHORrCC44V+aR6Ftzort912ZU+YLiSTuV8eaE=
+github.com/montanaflynn/stats v0.7.1/go.mod h1:etXPPgVO6n31NxCd9KQUMvCM+ve0ruNzt6R8Bnaayow=
github.com/mreiferson/go-httpclient v0.0.0-20160630210159-31f0106b4474/go.mod h1:OQA4XLvDbMgS8P0CevmM4m9Q3Jq4phKUzcocxuGJ5m8=
github.com/mreiferson/go-httpclient v0.0.0-20201222173833-5e475fde3a4d/go.mod h1:OQA4XLvDbMgS8P0CevmM4m9Q3Jq4phKUzcocxuGJ5m8=
github.com/muesli/reflow v0.3.0 h1:IFsN6K9NfGtjeggFP+68I4chLZV2yIKsXJFNZ+eWh6s=
@@ -1085,6 +1087,12 @@ github.com/xanzy/go-gitlab v0.107.0 h1:P2CT9Uy9yN9lJo3FLxpMZ4xj6uWcpnigXsjvqJ6nd
github.com/xanzy/go-gitlab v0.107.0/go.mod h1:wKNKh3GkYDMOsGmnfuX+ITCmDuSDWFO0G+C4AygL9RY=
github.com/xanzy/ssh-agent v0.3.3 h1:+/15pJfg/RsTxqYcX6fHqOXZwwMP+2VyYWJeWM2qQFM=
github.com/xanzy/ssh-agent v0.3.3/go.mod h1:6dzNDKs0J9rVPHPhaGCukekBHKqfl+L3KghI1Bc68Uw=
+github.com/xdg-go/pbkdf2 v1.0.0 h1:Su7DPu48wXMwC3bs7MCNG+z4FhcyEuz5dlvchbq0B0c=
+github.com/xdg-go/pbkdf2 v1.0.0/go.mod h1:jrpuAogTd400dnrH08LKmI/xc1MbPOebTwRqcT5RDeI=
+github.com/xdg-go/scram v1.1.2 h1:FHX5I5B4i4hKRVRBCFRxq1iQRej7WO3hhBuJf+UUySY=
+github.com/xdg-go/scram v1.1.2/go.mod h1:RT/sEzTbU5y00aCK8UOx6R7YryM0iF1N2MOmC3kKLN4=
+github.com/xdg-go/stringprep v1.0.4 h1:XLI/Ng3O1Atzq0oBs3TWm+5ZVgkq2aqdlvP9JtoZ6c8=
+github.com/xdg-go/stringprep v1.0.4/go.mod h1:mPGuuIYwz7CmR2bT9j4GbQqutWS1zV24gijq1dTyGkM=
github.com/xeipuuv/gojsonpointer v0.0.0-20180127040702-4e3ac2762d5f/go.mod h1:N2zxlSyiKSe5eX1tZViRH5QA0qijqEDrYZiPEAiq3wU=
github.com/xeipuuv/gojsonpointer v0.0.0-20190905194746-02993c407bfb h1:zGWFAtiMcyryUHoUjUJX0/lt1H2+i2Ka2n+D3DImSNo=
github.com/xeipuuv/gojsonpointer v0.0.0-20190905194746-02993c407bfb/go.mod h1:N2zxlSyiKSe5eX1tZViRH5QA0qijqEDrYZiPEAiq3wU=
@@ -1098,6 +1106,8 @@ github.com/xi2/xz v0.0.0-20171230120015-48954b6210f8/go.mod h1:HUYIGzjTL3rfEspMx
github.com/xiang90/probing v0.0.0-20190116061207-43a291ad63a2/go.mod h1:UETIi67q53MR2AWcXfiuqkDkRtnGDLqkBTpCHuJHxtU=
github.com/yl2chen/cidranger v1.0.2 h1:lbOWZVCG1tCRX4u24kuM1Tb4nHqWkDxwLdoS+SevawU=
github.com/yl2chen/cidranger v1.0.2/go.mod h1:9U1yz7WPYDwf0vpNWFaeRh0bjwz5RVgRy/9UEQfHl0g=
+github.com/youmark/pkcs8 v0.0.0-20240726163527-a2c0da244d78 h1:ilQV1hzziu+LLM3zUTJ0trRztfwgjqKnBWNtSRkbmwM=
+github.com/youmark/pkcs8 v0.0.0-20240726163527-a2c0da244d78/go.mod h1:aL8wCCfTfSfmXjznFBSZNN13rSJjlIOI1fUNAtF7rmI=
github.com/ysmood/fetchup v0.2.3 h1:ulX+SonA0Vma5zUFXtv52Kzip/xe7aj4vqT5AJwQ+ZQ=
github.com/ysmood/fetchup v0.2.3/go.mod h1:xhibcRKziSvol0H1/pj33dnKrYyI2ebIvz5cOOkYGns=
github.com/ysmood/goob v0.4.0 h1:HsxXhyLBeGzWXnqVKtmT9qM7EuVs/XOgkX7T6r1o1AQ=
@@ -1150,6 +1160,8 @@ go.etcd.io/bbolt v1.3.3/go.mod h1:IbVyRI1SCnLcuJnV2u8VeU0CEYM7e686BmAb1XKL+uU=
go.etcd.io/bbolt v1.3.10 h1:+BqfJTcCzTItrop8mq/lbzL8wSGtj94UO/3U31shqG0=
go.etcd.io/bbolt v1.3.10/go.mod h1:bK3UQLPJZly7IlNmV7uVHJDxfe5aK9Ll93e/74Y9oEQ=
go.etcd.io/etcd v0.0.0-20191023171146-3cf2f69b5738/go.mod h1:dnLIgRNXwCJa5e+c6mIZCrds/GIG4ncV9HhK5PX7jPg=
+go.mongodb.org/mongo-driver v1.17.0 h1:Hp4q2MCjvY19ViwimTs00wHi7G4yzxh4/2+nTx8r40k=
+go.mongodb.org/mongo-driver v1.17.0/go.mod h1:wwWm/+BuOddhcq3n68LKRmgk2wXzmF6s0SFOa0GINL4=
go.opencensus.io v0.20.1/go.mod h1:6WKK9ahsWS3RSO+PY9ZHZUfv2irvY6gN279GOPZjmmk=
go.opencensus.io v0.20.2/go.mod h1:6WKK9ahsWS3RSO+PY9ZHZUfv2irvY6gN279GOPZjmmk=
go.opencensus.io v0.21.0/go.mod h1:mSImk1erAIZhrmZN+AvHh14ztQfjbGwt4TtuofqLduU=
diff --git a/pkg/reporting/exporters/mongo/mongo.go b/pkg/reporting/exporters/mongo/mongo.go
new file mode 100644
index 0000000000..faf8bb579b
--- /dev/null
+++ b/pkg/reporting/exporters/mongo/mongo.go
@@ -0,0 +1,155 @@
+package mongo
+
+import (
+ "context"
+ "github.com/pkg/errors"
+ "github.com/projectdiscovery/gologger"
+ "github.com/projectdiscovery/nuclei/v3/pkg/output"
+ "go.mongodb.org/mongo-driver/mongo"
+ "net/url"
+ "os"
+ "strings"
+ "sync"
+
+ mongooptions "go.mongodb.org/mongo-driver/mongo/options"
+)
+
+type Exporter struct {
+ options *Options
+ mutex *sync.Mutex
+ rows []output.ResultEvent
+ collection *mongo.Collection
+ connection *mongo.Client
+}
+
+// Options contains the configuration options for MongoDB exporter client
+type Options struct {
+ // ConnectionString is the connection string to the MongoDB database
+ ConnectionString string `yaml:"connection-string"`
+ // CollectionName is the name of the MongoDB collection in which to store the results
+ CollectionName string `yaml:"collection-name"`
+ // OmitRaw excludes the Request and Response from the results (helps with filesize)
+ OmitRaw bool `yaml:"omit-raw"`
+ // BatchSize determines the number of results to be kept in memory before writing it to the database or 0 to
+ // persist all in memory and write all results at the end (default)
+ BatchSize int `yaml:"batch-size"`
+}
+
+// New creates a new MongoDB exporter integration client based on options.
+func New(options *Options) (*Exporter, error) {
+ exporter := &Exporter{
+ mutex: &sync.Mutex{},
+ options: options,
+ rows: []output.ResultEvent{},
+ }
+
+ // If the environment variable for the connection string is set, then use that instead. This allows for easier
+ // management of sensitive items such as credentials
+ envConnectionString := os.Getenv("MONGO_CONNECTION_STRING")
+ if envConnectionString != "" {
+ options.ConnectionString = envConnectionString
+ gologger.Info().Msgf("Using connection string from environment variable MONGO_CONNECTION_STRING")
+ }
+
+ // Create the connection to the database
+ clientOptions := mongooptions.Client().ApplyURI(options.ConnectionString)
+
+ // Create a new client and connect to the MongoDB server
+ client, err := mongo.Connect(context.TODO(), clientOptions)
+ if err != nil {
+ gologger.Error().Msgf("Error creating MongoDB client: %s", err)
+ return nil, err
+ }
+
+ // Ensure the connection is valid
+ err = client.Ping(context.Background(), nil)
+ if err != nil {
+ gologger.Error().Msgf("Error connecting to MongoDB: %s", err)
+ return nil, err
+ }
+
+ // Get the database from the connection string to set the database and collection
+ parsed, err := url.Parse(options.ConnectionString)
+ if err != nil {
+ gologger.Error().Msgf("Error parsing connection string: %s", options.ConnectionString)
+ return nil, err
+ }
+
+ databaseName := strings.TrimPrefix(parsed.Path, "/")
+
+ if databaseName == "" {
+ return nil, errors.New("error getting database name from connection string")
+ }
+
+ exporter.connection = client
+ exporter.collection = client.Database(databaseName).Collection(options.CollectionName)
+
+ return exporter, nil
+}
+
+// Export writes a result document to the configured MongoDB collection
+// in the database configured by the connection string
+func (exporter *Exporter) Export(event *output.ResultEvent) error {
+ exporter.mutex.Lock()
+ defer exporter.mutex.Unlock()
+
+ if exporter.options.OmitRaw {
+ event.Request = ""
+ event.Response = ""
+ }
+
+ // Add the row to the queue to be processed
+ exporter.rows = append(exporter.rows, *event)
+
+ // If the batch size is greater than 0 and the number of rows has reached the batch, flush it to the database
+ if exporter.options.BatchSize > 0 && len(exporter.rows) >= exporter.options.BatchSize {
+ err := exporter.WriteRows()
+ if err != nil {
+ // The error is already logged, return it to bubble up to the caller
+ return err
+ }
+ }
+
+ return nil
+}
+
+// WriteRows writes all rows from the rows list to the MongoDB collection and removes them from the list
+func (exporter *Exporter) WriteRows() error {
+ // Loop through the rows and write them, removing them as they're entered
+ for len(exporter.rows) > 0 {
+ data := exporter.rows[0]
+
+ // Write the data to the database
+ _, err := exporter.collection.InsertOne(context.TODO(), data)
+ if err != nil {
+ gologger.Fatal().Msgf("Error inserting record into MongoDB collection: %s", err)
+ return err
+ }
+
+ // Remove the item from the list
+ exporter.rows = exporter.rows[1:]
+ }
+
+ return nil
+}
+
+func (exporter *Exporter) Close() error {
+ exporter.mutex.Lock()
+ defer exporter.mutex.Unlock()
+
+ // Write all pending rows
+ err := exporter.WriteRows()
+ if err != nil {
+ // The error is already logged, return it to bubble up to the caller
+ return err
+ }
+
+ // Close the database connection
+ err = exporter.connection.Disconnect(context.TODO())
+ if err != nil {
+ gologger.Error().Msgf("Error disconnecting from MongoDB: %s", err)
+ return err
+ }
+
+ return nil
+}
diff --git a/pkg/reporting/options.go b/pkg/reporting/options.go
index c5090de014..bda9b6c28d 100644
--- a/pkg/reporting/options.go
+++ b/pkg/reporting/options.go
@@ -5,6 +5,7 @@ import (
"github.com/projectdiscovery/nuclei/v3/pkg/reporting/exporters/jsonexporter"
"github.com/projectdiscovery/nuclei/v3/pkg/reporting/exporters/jsonl"
"github.com/projectdiscovery/nuclei/v3/pkg/reporting/exporters/markdown"
+ "github.com/projectdiscovery/nuclei/v3/pkg/reporting/exporters/mongo"
"github.com/projectdiscovery/nuclei/v3/pkg/reporting/exporters/sarif"
"github.com/projectdiscovery/nuclei/v3/pkg/reporting/exporters/splunk"
"github.com/projectdiscovery/nuclei/v3/pkg/reporting/trackers/filters"
@@ -44,6 +45,8 @@ type Options struct {
JSONExporter *jsonexporter.Options `yaml:"json"`
// JSONLExporter contains configuration options for JSONL Exporter Module
JSONLExporter *jsonl.Options `yaml:"jsonl"`
+ // MongoDBExporter containers the configuration options for the MongoDB Exporter Module
+ MongoDBExporter *mongo.Options `yaml:"mongodb"`
HttpClient *retryablehttp.Client `yaml:"-"`
OmitRaw bool `yaml:"-"`
diff --git a/pkg/reporting/reporting.go b/pkg/reporting/reporting.go
index c6a7d63e10..ddc5428636 100644
--- a/pkg/reporting/reporting.go
+++ b/pkg/reporting/reporting.go
@@ -2,6 +2,7 @@ package reporting
import (
"fmt"
+ "github.com/projectdiscovery/nuclei/v3/pkg/reporting/exporters/mongo"
"os"
"strings"
"sync/atomic"
@@ -166,6 +167,13 @@ func New(options *Options, db string, doNotDedupe bool) (Client, error) {
}
client.exporters = append(client.exporters, exporter)
}
+ if options.MongoDBExporter != nil {
+ exporter, err := mongo.New(options.MongoDBExporter)
+ if err != nil {
+ return nil, errorutil.NewWithErr(err).Wrap(ErrExportClientCreation)
+ }
+ client.exporters = append(client.exporters, exporter)
+ }
if doNotDedupe {
return client, nil
@@ -212,6 +220,7 @@ func CreateConfigIfNotExists() error {
SplunkExporter: &splunk.Options{},
JSONExporter: &json_exporter.Options{},
JSONLExporter: &jsonl.Options{},
+ MongoDBExporter: &mongo.Options{},
}
reportingFile, err := os.Create(reportingConfig)
if err != nil {
From 888a732fbc9b16c7776f78f147d9277641fc61ca Mon Sep 17 00:00:00 2001
From: Danny Shemesh
Date: Mon, 14 Oct 2024 11:48:59 +0300
Subject: [PATCH 230/283] Unlock memguard global change mutex only when locked
(#5714)
---
pkg/protocols/common/protocolstate/memguardian.go | 6 ++----
1 file changed, 2 insertions(+), 4 deletions(-)
diff --git a/pkg/protocols/common/protocolstate/memguardian.go b/pkg/protocols/common/protocolstate/memguardian.go
index 1db1e0a775..b5c8cffc75 100644
--- a/pkg/protocols/common/protocolstate/memguardian.go
+++ b/pkg/protocols/common/protocolstate/memguardian.go
@@ -77,9 +77,8 @@ var muGlobalChange sync.Mutex
// Global setting
func GlobalGuardBytesBufferAlloc() error {
- if muGlobalChange.TryLock() {
+ if !muGlobalChange.TryLock() {
return nil
-
}
defer muGlobalChange.Unlock()
@@ -95,9 +94,8 @@ func GlobalGuardBytesBufferAlloc() error {
// Global setting
func GlobalRestoreBytesBufferAlloc() {
- if muGlobalChange.TryLock() {
+ if !muGlobalChange.TryLock() {
return
-
}
defer muGlobalChange.Unlock()
From d68af67e6ea08c9df13f44af8e9fbd9dc378faf2 Mon Sep 17 00:00:00 2001
From: Dwi Siswanto
Date: Mon, 14 Oct 2024 16:23:36 +0700
Subject: [PATCH 231/283] feat(nuclei): generate trace file when using
`profile-mem` (#5690)
* feat(nuclei): generate trace file when using `profile-mem`
Signed-off-by: Dwi Siswanto
* docs(DESIGN): dynamically grep mod path
Signed-off-by: Dwi Siswanto
---------
Signed-off-by: Dwi Siswanto
---
.gitignore | 7 ++++++-
DESIGN.md | 43 +++++++++++++++++++++++++++++++++----------
cmd/nuclei/main.go | 40 ++++++++++++++++++++++++++++++----------
3 files changed, 69 insertions(+), 21 deletions(-)
diff --git a/.gitignore b/.gitignore
index 146b0a892c..6145103342 100644
--- a/.gitignore
+++ b/.gitignore
@@ -42,4 +42,9 @@ pkg/protocols/common/helpers/deserialization/testdata/ValueObject2.ser
vendor
# Headless `screenshot` action
-*.png
\ No newline at end of file
+*.png
+
+# Profiling & tracing
+*.prof
+*.pprof
+*.trace
\ No newline at end of file
diff --git a/DESIGN.md b/DESIGN.md
index 9d92e28f8d..af0e27baf9 100644
--- a/DESIGN.md
+++ b/DESIGN.md
@@ -457,26 +457,49 @@ func (template *Template) compileProtocolRequests(options protocols.ExecuterOpti
That's it, you've added a new protocol to Nuclei. The next good step would be to write integration tests which are described in `integration-tests` and `cmd/integration-tests` directories.
-## Profiling Instructions
+## Profiling and Tracing
-To enable dumping of Memory profiling data, `-profile-mem` flag can be used along with path to a file. This writes a pprof formatted file which can be used for investigate resource usage with `pprof` tool.
+To analyze Nuclei's performance and resource usage, you can generate memory profiles and trace files using the `-profile-mem` flag:
-```console
-$ nuclei -t nuclei-templates/ -u https://example.com -profile-mem mem.pprof
+```bash
+nuclei -t nuclei-templates/ -u https://example.com -profile-mem=nuclei-$(git describe --tags)
```
-To view profile data in pprof, first install pprof. Then run the below command -
+This command creates two files:
-```console
-$ go tool pprof mem.pprof
+* `nuclei.prof`: Memory (heap) profile
+* `nuclei.trace`: Execution trace
+
+### Analyzing the Memory Profile
+
+1. View the profile in the terminal:
+
+```bash
+go tool pprof nuclei.prof
+```
+
+2. Display top memory consumers:
+
+```bash
+go tool pprof -top nuclei.prof | grep "$(go list -m)" | head -10
```
-To open a web UI on a port to visualize debug data, the below command can be used.
+3. Visualize the profile in a web browser:
-```console
-$ go tool pprof -http=:8081 mem.pprof
+```bash
+go tool pprof -http=:$(shuf -i 1000-99999 -n 1) nuclei.prof
```
+### Analyzing the Trace File
+
+To examine the execution trace:
+
+```bash
+go tool trace nuclei.trace
+```
+
+These tools help identify performance bottlenecks and memory leaks, allowing for targeted optimizations of Nuclei's codebase.
+
## Project Structure
- [pkg/reporting](./pkg/reporting) - Reporting modules for nuclei.
diff --git a/cmd/nuclei/main.go b/cmd/nuclei/main.go
index 8fe5b29573..caac7e5546 100644
--- a/cmd/nuclei/main.go
+++ b/cmd/nuclei/main.go
@@ -9,6 +9,7 @@ import (
"path/filepath"
"runtime"
"runtime/pprof"
+ "runtime/trace"
"strings"
"time"
@@ -103,21 +104,40 @@ func main() {
return
}
- // Profiling related code
+ // Profiling & tracing related code
if memProfile != "" {
- f, err := os.Create(memProfile)
+ memProfile = strings.TrimSuffix(memProfile, filepath.Ext(memProfile)) + ".prof"
+ memProfileFile, err := os.Create(memProfile)
if err != nil {
- gologger.Fatal().Msgf("profile: could not create memory profile %q: %v", memProfile, err)
+ gologger.Fatal().Msgf("profile: could not create memory profile %q file: %v", memProfile, err)
}
- old := runtime.MemProfileRate
+
+ traceFilepath := strings.TrimSuffix(memProfile, filepath.Ext(memProfile)) + ".trace"
+ traceFile, err := os.Create(traceFilepath)
+ if err != nil {
+ gologger.Fatal().Msgf("profile: could not create trace %q file: %v", traceFilepath, err)
+ }
+
+ oldMemProfileRate := runtime.MemProfileRate
runtime.MemProfileRate = 4096
- gologger.Print().Msgf("profile: memory profiling enabled (rate %d), %s", runtime.MemProfileRate, memProfile)
+
+ // Start tracing
+ if err := trace.Start(traceFile); err != nil {
+ gologger.Fatal().Msgf("profile: could not start trace: %v", err)
+ }
defer func() {
- _ = pprof.Lookup("heap").WriteTo(f, 0)
- f.Close()
- runtime.MemProfileRate = old
- gologger.Print().Msgf("profile: memory profiling disabled, %s", memProfile)
+ // Start CPU profiling
+ if err := pprof.WriteHeapProfile(memProfileFile); err != nil {
+ gologger.Fatal().Msgf("profile: could not start CPU profile: %v", err)
+ }
+ memProfileFile.Close()
+ traceFile.Close()
+ trace.Stop()
+ runtime.MemProfileRate = oldMemProfileRate
+
+ gologger.Info().Msgf("Memory profile saved at %q", memProfile)
+ gologger.Info().Msgf("Traced at %q", traceFilepath)
}()
}
@@ -402,7 +422,7 @@ on extensive configurability, massive extensibility and ease of use.`)
flagSet.CallbackVar(printVersion, "version", "show nuclei version"),
flagSet.BoolVarP(&options.HangMonitor, "hang-monitor", "hm", false, "enable nuclei hang monitoring"),
flagSet.BoolVarP(&options.Verbose, "verbose", "v", false, "show verbose output"),
- flagSet.StringVar(&memProfile, "profile-mem", "", "optional nuclei memory profile dump file"),
+ flagSet.StringVar(&memProfile, "profile-mem", "", "generate memory (heap) profile & trace files"),
flagSet.BoolVar(&options.VerboseVerbose, "vv", false, "display templates loaded for scan"),
flagSet.BoolVarP(&options.ShowVarDump, "show-var-dump", "svd", false, "show variables dump for debugging"),
flagSet.BoolVarP(&options.EnablePprof, "enable-pprof", "ep", false, "enable pprof debugging server"),
From 4ef058758ce29807669b5b350b8dc10d43e7f9c8 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Do=C4=9Fan=20Can=20Bak=C4=B1r?=
Date: Mon, 14 Oct 2024 15:53:53 +0300
Subject: [PATCH 232/283] fix template loading logic
---
lib/sdk.go | 1 +
1 file changed, 1 insertion(+)
diff --git a/lib/sdk.go b/lib/sdk.go
index daeb68c14b..9bd46f4766 100644
--- a/lib/sdk.go
+++ b/lib/sdk.go
@@ -100,6 +100,7 @@ func (e *NucleiEngine) LoadAllTemplates() error {
return errorutil.New("Could not create loader client: %s\n", err)
}
e.store.Load()
+ e.templatesLoaded = true
return nil
}
From aab2cadb64e32d213586fdea25c8b6192007482d Mon Sep 17 00:00:00 2001
From: chuu
Date: Mon, 14 Oct 2024 06:52:52 -0700
Subject: [PATCH 233/283] fix: input helper in executor options (#5712)
---
lib/sdk_private.go | 2 ++
1 file changed, 2 insertions(+)
diff --git a/lib/sdk_private.go b/lib/sdk_private.go
index cacd9a1ca1..f63c7b1c34 100644
--- a/lib/sdk_private.go
+++ b/lib/sdk_private.go
@@ -3,6 +3,7 @@ package nuclei
import (
"context"
"fmt"
+ "github.com/projectdiscovery/nuclei/v3/pkg/input"
"strings"
"sync"
"time"
@@ -171,6 +172,7 @@ func (e *NucleiEngine) init(ctx context.Context) error {
ResumeCfg: types.NewResumeCfg(),
Browser: e.browserInstance,
Parser: e.parser,
+ InputHelper: input.NewHelper(),
}
if len(e.opts.SecretsFile) > 0 {
authTmplStore, err := runner.GetAuthTmplStore(*e.opts, e.catalog, e.executerOpts)
From cc5c5509dc8d87f8b0779b0dfa1e1b9f6f31b5fb Mon Sep 17 00:00:00 2001
From: Dwi Siswanto
Date: Mon, 14 Oct 2024 20:55:46 +0700
Subject: [PATCH 234/283] feat: global matchers (#5701)
* feat: global matchers
Signed-off-by: Dwi Siswanto
Co-authored-by: Ice3man543
* feat(globalmatchers): make `Callback` as type
Signed-off-by: Dwi Siswanto
* feat: update `passive` term to `(matchers-)static`
Signed-off-by: Dwi Siswanto
* feat(globalmatchers): add `origin-template-*` event
also use `Set` method instead of `maps.Clone`
Signed-off-by: Dwi Siswanto
* feat: update `matchers-static` term to `global-matchers`
Signed-off-by: Dwi Siswanto
* feat(globalmatchers): clone event before `operator.Execute`
Signed-off-by: Dwi Siswanto
* fix(tmplexec): don't store `matched` on `global-matchers` templ
This will end up generating 2 events from the same
`scan.ScanContext` if one of the templates has
`global-matchers` enabled. This way, non-
`global-matchers` templates can enter the
`writeFailureCallback` func to log failure output.
Signed-off-by: Dwi Siswanto
* feat(globalmatchers): initializes `requests` on `New`
Signed-off-by: Dwi Siswanto
* feat(globalmatchers): add `hasStorage` method
Signed-off-by: Dwi Siswanto
* refactor(templates): rename global matchers checks method
Signed-off-by: Dwi Siswanto
* fix(loader): handle nil `templates.Template` pointer
Signed-off-by: Dwi Siswanto
---------
Signed-off-by: Dwi Siswanto
Co-authored-by: Ice3man543
---
internal/runner/runner.go | 2 +
pkg/catalog/loader/loader.go | 14 ++++
pkg/output/format_screen.go | 5 ++
pkg/output/output.go | 3 +
.../common/globalmatchers/globalmatchers.go | 84 +++++++++++++++++++
pkg/protocols/http/http.go | 3 +
pkg/protocols/http/operators.go | 5 ++
pkg/protocols/http/request.go | 11 ++-
pkg/protocols/protocols.go | 3 +
pkg/templates/compile.go | 32 +++++++
pkg/tmplexec/exec.go | 10 ++-
11 files changed, 170 insertions(+), 2 deletions(-)
create mode 100644 pkg/protocols/common/globalmatchers/globalmatchers.go
diff --git a/internal/runner/runner.go b/internal/runner/runner.go
index 0b6da592d3..516bd7ca50 100644
--- a/internal/runner/runner.go
+++ b/internal/runner/runner.go
@@ -48,6 +48,7 @@ import (
"github.com/projectdiscovery/nuclei/v3/pkg/protocols"
"github.com/projectdiscovery/nuclei/v3/pkg/protocols/common/automaticscan"
"github.com/projectdiscovery/nuclei/v3/pkg/protocols/common/contextargs"
+ "github.com/projectdiscovery/nuclei/v3/pkg/protocols/common/globalmatchers"
"github.com/projectdiscovery/nuclei/v3/pkg/protocols/common/hosterrorscache"
"github.com/projectdiscovery/nuclei/v3/pkg/protocols/common/interactsh"
"github.com/projectdiscovery/nuclei/v3/pkg/protocols/common/protocolinit"
@@ -475,6 +476,7 @@ func (r *Runner) RunEnumeration() error {
TemporaryDirectory: r.tmpDir,
Parser: r.parser,
FuzzParamsFrequency: fuzzFreqCache,
+ GlobalMatchers: globalmatchers.New(),
}
if config.DefaultConfig.IsDebugArgEnabled(config.DebugExportURLPattern) {
diff --git a/pkg/catalog/loader/loader.go b/pkg/catalog/loader/loader.go
index 48f14a4054..94818020b7 100644
--- a/pkg/catalog/loader/loader.go
+++ b/pkg/catalog/loader/loader.go
@@ -353,6 +353,20 @@ func (store *Store) areWorkflowOrTemplatesValid(filteredTemplatePaths map[string
if isParsingError("Error occurred parsing template %s: %s\n", templatePath, err) {
areTemplatesValid = false
}
+ } else if template == nil {
+ // NOTE(dwisiswant0): possibly global matchers template.
+ // This could definitely be handled better, for example by returning an
+ // `ErrGlobalMatchersTemplate` during `templates.Parse` and checking it
+ // with `errors.Is`.
+ //
+ // However, I’m not sure if every reference to it should be handled
+ // that way. Returning a `templates.Template` pointer would mean it’s
+ // an active template (sending requests), and adding a specific field
+ // like `isGlobalMatchers` in `templates.Template` (then checking it
+ // with a `*templates.Template.IsGlobalMatchersEnabled` method) would
+ // just introduce more unknown issues - like during template
+ // clustering, AFAIK.
+ continue
} else {
if existingTemplatePath, found := templateIDPathMap[template.ID]; !found {
templateIDPathMap[template.ID] = templatePath
diff --git a/pkg/output/format_screen.go b/pkg/output/format_screen.go
index 9d0f03efaf..3cdcec1e15 100644
--- a/pkg/output/format_screen.go
+++ b/pkg/output/format_screen.go
@@ -39,6 +39,11 @@ func (w *StandardWriter) formatScreen(output *ResultEvent) []byte {
}
}
+ if output.GlobalMatchers {
+ builder.WriteString("] [")
+ builder.WriteString(w.aurora.BrightMagenta("global").String())
+ }
+
builder.WriteString("] [")
builder.WriteString(w.aurora.BrightBlue(output.Type).String())
builder.WriteString("] ")
diff --git a/pkg/output/output.go b/pkg/output/output.go
index fbc9f71306..9d84fd20fc 100644
--- a/pkg/output/output.go
+++ b/pkg/output/output.go
@@ -184,6 +184,9 @@ type ResultEvent struct {
MatcherStatus bool `json:"matcher-status"`
// Lines is the line count for the specified match
Lines []int `json:"matched-line,omitempty"`
+ // GlobalMatchers identifies whether the matches was detected in the response
+ // of another template's result event
+ GlobalMatchers bool `json:"global-matchers,omitempty"`
// IssueTrackers is the metadata for issue trackers
IssueTrackers map[string]IssueTrackerMetadata `json:"issue_trackers,omitempty"`
diff --git a/pkg/protocols/common/globalmatchers/globalmatchers.go b/pkg/protocols/common/globalmatchers/globalmatchers.go
new file mode 100644
index 0000000000..321759f927
--- /dev/null
+++ b/pkg/protocols/common/globalmatchers/globalmatchers.go
@@ -0,0 +1,84 @@
+package globalmatchers
+
+import (
+ "maps"
+
+ "github.com/projectdiscovery/nuclei/v3/pkg/model"
+ "github.com/projectdiscovery/nuclei/v3/pkg/operators"
+ "github.com/projectdiscovery/nuclei/v3/pkg/output"
+)
+
+// Storage is a struct that holds the global matchers
+type Storage struct {
+ requests []*Item
+}
+
+// Callback is called when a global matcher is matched.
+// It receives internal event & result of the operator execution.
+type Callback func(event output.InternalEvent, result *operators.Result)
+
+// Item is a struct that holds the global matchers
+// details for a template
+type Item struct {
+ TemplateID string
+ TemplatePath string
+ TemplateInfo model.Info
+ Operators []*operators.Operators
+}
+
+// New creates a new storage for global matchers
+func New() *Storage {
+ return &Storage{requests: make([]*Item, 0)}
+}
+
+// hasStorage checks if the Storage is initialized
+func (s *Storage) hasStorage() bool {
+ return s != nil
+}
+
+// AddOperator adds a new operator to the global matchers
+func (s *Storage) AddOperator(item *Item) {
+ if !s.hasStorage() {
+ return
+ }
+
+ s.requests = append(s.requests, item)
+}
+
+// HasMatchers returns true if we have global matchers
+func (s *Storage) HasMatchers() bool {
+ if !s.hasStorage() {
+ return false
+ }
+
+ return len(s.requests) > 0
+}
+
+// Match matches the global matchers against the response
+func (s *Storage) Match(
+ event output.InternalEvent,
+ matchFunc operators.MatchFunc,
+ extractFunc operators.ExtractFunc,
+ isDebug bool,
+ callback Callback,
+) {
+ for _, item := range s.requests {
+ for _, operator := range item.Operators {
+ newEvent := maps.Clone(event)
+ newEvent.Set("origin-template-id", event["template-id"])
+ newEvent.Set("origin-template-info", event["template-info"])
+ newEvent.Set("origin-template-path", event["template-path"])
+ newEvent.Set("template-id", item.TemplateID)
+ newEvent.Set("template-info", item.TemplateInfo)
+ newEvent.Set("template-path", item.TemplatePath)
+ newEvent.Set("global-matchers", true)
+
+ result, matched := operator.Execute(newEvent, matchFunc, extractFunc, isDebug)
+ if !matched {
+ continue
+ }
+
+ callback(newEvent, result)
+ }
+ }
+}
diff --git a/pkg/protocols/http/http.go b/pkg/protocols/http/http.go
index 844bf8c579..bd089edd47 100644
--- a/pkg/protocols/http/http.go
+++ b/pkg/protocols/http/http.go
@@ -223,6 +223,9 @@ type Request struct {
// FuzzPreConditionOperator is the operator between multiple PreConditions for fuzzing Default is OR
FuzzPreConditionOperator string `yaml:"pre-condition-operator,omitempty" json:"pre-condition-operator,omitempty" jsonschema:"title=condition between the filters,description=Operator to use between multiple per-conditions,enum=and,enum=or"`
fuzzPreConditionOperator matchers.ConditionType `yaml:"-" json:"-"`
+ // description: |
+ // GlobalMatchers marks matchers as static and applies globally to all result events from other templates
+ GlobalMatchers bool `yaml:"global-matchers,omitempty" json:"global-matchers,omitempty" jsonschema:"title=global matchers,description=marks matchers as static and applies globally to all result events from other templates"`
}
func (e Request) JSONSchemaExtend(schema *jsonschema.Schema) {
diff --git a/pkg/protocols/http/operators.go b/pkg/protocols/http/operators.go
index d630bfd8b0..9e7d58af0a 100644
--- a/pkg/protocols/http/operators.go
+++ b/pkg/protocols/http/operators.go
@@ -166,6 +166,10 @@ func (request *Request) MakeResultEventItem(wrapped *output.InternalWrappedEvent
if types.ToString(wrapped.InternalEvent["path"]) != "" {
fields.Path = types.ToString(wrapped.InternalEvent["path"])
}
+ var isGlobalMatchers bool
+ if value, ok := wrapped.InternalEvent["global-matchers"]; ok {
+ isGlobalMatchers = value.(bool)
+ }
data := &output.ResultEvent{
TemplateID: types.ToString(wrapped.InternalEvent["template-id"]),
TemplatePath: types.ToString(wrapped.InternalEvent["template-path"]),
@@ -183,6 +187,7 @@ func (request *Request) MakeResultEventItem(wrapped *output.InternalWrappedEvent
Timestamp: time.Now(),
MatcherStatus: true,
IP: fields.Ip,
+ GlobalMatchers: isGlobalMatchers,
Request: types.ToString(wrapped.InternalEvent["request"]),
Response: request.truncateResponse(wrapped.InternalEvent["response"]),
CURLCommand: types.ToString(wrapped.InternalEvent["curl-command"]),
diff --git a/pkg/protocols/http/request.go b/pkg/protocols/http/request.go
index 994e065582..f74020eefb 100644
--- a/pkg/protocols/http/request.go
+++ b/pkg/protocols/http/request.go
@@ -973,13 +973,22 @@ func (request *Request) executeRequest(input *contextargs.Context, generatedRequ
// prune signature internal values if any
request.pruneSignatureInternalValues(generatedRequest.meta)
- event := eventcreator.CreateEventWithAdditionalOptions(request, generators.MergeMaps(generatedRequest.dynamicValues, finalEvent), request.options.Options.Debug || request.options.Options.DebugResponse, func(internalWrappedEvent *output.InternalWrappedEvent) {
+ interimEvent := generators.MergeMaps(generatedRequest.dynamicValues, finalEvent)
+ isDebug := request.options.Options.Debug || request.options.Options.DebugResponse
+ event := eventcreator.CreateEventWithAdditionalOptions(request, interimEvent, isDebug, func(internalWrappedEvent *output.InternalWrappedEvent) {
internalWrappedEvent.OperatorsResult.PayloadValues = generatedRequest.meta
})
+
if hasInteractMatchers {
event.UsesInteractsh = true
}
+ if request.options.GlobalMatchers.HasMatchers() {
+ request.options.GlobalMatchers.Match(interimEvent, request.Match, request.Extract, isDebug, func(event output.InternalEvent, result *operators.Result) {
+ callback(eventcreator.CreateEventWithOperatorResults(request, event, result))
+ })
+ }
+
// if requrlpattern is enabled, only then it is reflected in result event else it is empty string
// consult @Ice3man543 before changing this logic (context: vuln_hash)
if request.options.ExportReqURLPattern {
diff --git a/pkg/protocols/protocols.go b/pkg/protocols/protocols.go
index a9d50c1481..9ead70321d 100644
--- a/pkg/protocols/protocols.go
+++ b/pkg/protocols/protocols.go
@@ -25,6 +25,7 @@ import (
"github.com/projectdiscovery/nuclei/v3/pkg/progress"
"github.com/projectdiscovery/nuclei/v3/pkg/projectfile"
"github.com/projectdiscovery/nuclei/v3/pkg/protocols/common/contextargs"
+ "github.com/projectdiscovery/nuclei/v3/pkg/protocols/common/globalmatchers"
"github.com/projectdiscovery/nuclei/v3/pkg/protocols/common/hosterrorscache"
"github.com/projectdiscovery/nuclei/v3/pkg/protocols/common/interactsh"
"github.com/projectdiscovery/nuclei/v3/pkg/protocols/common/utils/excludematchers"
@@ -126,6 +127,8 @@ type ExecutorOptions struct {
// ExportReqURLPattern exports the request URL pattern
// in ResultEvent it contains the exact url pattern (ex: {{BaseURL}}/{{randstr}}/xyz) used in the request
ExportReqURLPattern bool
+ // GlobalMatchers is the storage for global matchers with http passive templates
+ GlobalMatchers *globalmatchers.Storage
}
// todo: centralizing components is not feasible with current clogged architecture
diff --git a/pkg/templates/compile.go b/pkg/templates/compile.go
index 01af3a999b..b4005afe81 100644
--- a/pkg/templates/compile.go
+++ b/pkg/templates/compile.go
@@ -20,6 +20,7 @@ import (
"github.com/projectdiscovery/nuclei/v3/pkg/operators"
"github.com/projectdiscovery/nuclei/v3/pkg/protocols"
"github.com/projectdiscovery/nuclei/v3/pkg/protocols/common/generators"
+ "github.com/projectdiscovery/nuclei/v3/pkg/protocols/common/globalmatchers"
"github.com/projectdiscovery/nuclei/v3/pkg/protocols/offlinehttp"
"github.com/projectdiscovery/nuclei/v3/pkg/templates/signer"
"github.com/projectdiscovery/nuclei/v3/pkg/tmplexec"
@@ -81,6 +82,18 @@ func Parse(filePath string, preprocessor Preprocessor, options protocols.Executo
if err != nil {
return nil, err
}
+ if template.isGlobalMatchersEnabled() {
+ item := &globalmatchers.Item{
+ TemplateID: template.ID,
+ TemplatePath: filePath,
+ TemplateInfo: template.Info,
+ }
+ for _, request := range template.RequestsHTTP {
+ item.Operators = append(item.Operators, request.CompiledOperators)
+ }
+ options.GlobalMatchers.AddOperator(item)
+ return nil, nil
+ }
// Compile the workflow request
if len(template.Workflows) > 0 {
compiled := &template.Workflow
@@ -96,6 +109,25 @@ func Parse(filePath string, preprocessor Preprocessor, options protocols.Executo
return template, nil
}
+// isGlobalMatchersEnabled checks if any of requests in the template
+// have global matchers enabled. It iterates through all requests and
+// returns true if at least one request has global matchers enabled;
+// otherwise, it returns false.
+//
+// Note: This method only checks the `RequestsHTTP`
+// field of the template, which is specific to http-protocol-based
+// templates.
+//
+// TODO: support all protocols.
+func (template *Template) isGlobalMatchersEnabled() bool {
+ for _, request := range template.RequestsHTTP {
+ if request.GlobalMatchers {
+ return true
+ }
+ }
+ return false
+}
+
// parseSelfContainedRequests parses the self contained template requests.
func (template *Template) parseSelfContainedRequests() {
if template.Signature.Value.String() != "" {
diff --git a/pkg/tmplexec/exec.go b/pkg/tmplexec/exec.go
index 279d03d849..0caefe6024 100644
--- a/pkg/tmplexec/exec.go
+++ b/pkg/tmplexec/exec.go
@@ -174,7 +174,15 @@ func (e *TemplateExecuter) Execute(ctx *scan.ScanContext) (bool, error) {
if !event.HasOperatorResult() && event.InternalEvent != nil {
lastMatcherEvent = event
} else {
- if writer.WriteResult(event, e.options.Output, e.options.Progress, e.options.IssuesClient) {
+ var isGlobalMatchers bool
+ isGlobalMatchers, _ = event.InternalEvent["global-matchers"].(bool)
+ // NOTE(dwisiswant0): Don't store `matched` on a `global-matchers` template.
+ // This will end up generating 2 events from the same `scan.ScanContext` if
+ // one of the templates has `global-matchers` enabled. This way,
+ // non-`global-matchers` templates can enter the `writeFailureCallback`
+ // func to log failure output.
+ wr := writer.WriteResult(event, e.options.Output, e.options.Progress, e.options.IssuesClient)
+ if wr && !isGlobalMatchers {
matched.Store(true)
} else {
lastMatcherEvent = event
From 53f56e179d6a5f254a442dcc9690153a75b9df10 Mon Sep 17 00:00:00 2001
From: ghost
Date: Mon, 14 Oct 2024 13:56:50 +0000
Subject: [PATCH 235/283] Auto Generate Syntax Docs + JSONSchema [Mon Oct 14
13:56:50 UTC 2024] :robot:
---
SYNTAX-REFERENCE.md | 13 +++++++++++++
nuclei-jsonschema.json | 5 +++++
pkg/templates/templates_doc.go | 7 ++++++-
3 files changed, 24 insertions(+), 1 deletion(-)
diff --git a/SYNTAX-REFERENCE.md b/SYNTAX-REFERENCE.md
index 394f0a0684..c448d251d3 100755
--- a/SYNTAX-REFERENCE.md
+++ b/SYNTAX-REFERENCE.md
@@ -1651,6 +1651,19 @@ FuzzPreConditionOperator is the operator between multiple PreConditions for fuzz
+
+
+global-matchers
bool
+
+
+
+
+GlobalMatchers marks matchers as static and applies globally to all result events from other templates
+
+
+
+
+
diff --git a/nuclei-jsonschema.json b/nuclei-jsonschema.json
index b4ed1d2982..8a8d5dd1ba 100644
--- a/nuclei-jsonschema.json
+++ b/nuclei-jsonschema.json
@@ -889,6 +889,11 @@
],
"title": "condition between the filters",
"description": "Operator to use between multiple per-conditions"
+ },
+ "global-matchers": {
+ "type": "boolean",
+ "title": "global matchers",
+ "description": "marks matchers as static and applies globally to all result events from other templates"
}
},
"additionalProperties": false,
diff --git a/pkg/templates/templates_doc.go b/pkg/templates/templates_doc.go
index b1422c8333..9a7fc97a37 100644
--- a/pkg/templates/templates_doc.go
+++ b/pkg/templates/templates_doc.go
@@ -459,7 +459,7 @@ func init() {
Value: "HTTP response headers in name:value format",
},
}
- HTTPRequestDoc.Fields = make([]encoder.Doc, 36)
+ HTTPRequestDoc.Fields = make([]encoder.Doc, 37)
HTTPRequestDoc.Fields[0].Name = "path"
HTTPRequestDoc.Fields[0].Type = "[]string"
HTTPRequestDoc.Fields[0].Note = ""
@@ -668,6 +668,11 @@ func init() {
HTTPRequestDoc.Fields[35].Note = ""
HTTPRequestDoc.Fields[35].Description = "FuzzPreConditionOperator is the operator between multiple PreConditions for fuzzing Default is OR"
HTTPRequestDoc.Fields[35].Comments[encoder.LineComment] = "FuzzPreConditionOperator is the operator between multiple PreConditions for fuzzing Default is OR"
+ HTTPRequestDoc.Fields[36].Name = "global-matchers"
+ HTTPRequestDoc.Fields[36].Type = "bool"
+ HTTPRequestDoc.Fields[36].Note = ""
+ HTTPRequestDoc.Fields[36].Description = "GlobalMatchers marks matchers as static and applies globally to all result events from other templates"
+ HTTPRequestDoc.Fields[36].Comments[encoder.LineComment] = "GlobalMatchers marks matchers as static and applies globally to all result events from other templates"
GENERATORSAttackTypeHolderDoc.Type = "generators.AttackTypeHolder"
GENERATORSAttackTypeHolderDoc.Comments[encoder.LineComment] = " AttackTypeHolder is used to hold internal type of the protocol"
From 2c832f559045423bbc9bd1e9b87d85928f429e5c Mon Sep 17 00:00:00 2001
From: Dwi Siswanto
Date: Mon, 14 Oct 2024 21:01:36 +0700
Subject: [PATCH 236/283] refactor(vardump): use `godump` lib (#5676)
* refactor(vardump): use `godump` lib
also increate limit char to `255`.
Signed-off-by: Dwi Siswanto
* feat(vardump): add global var `Limit`
Signed-off-by: Dwi Siswanto
* chore(protocols): rm newline
Signed-off-by: Dwi Siswanto
* feat(types): add `VarDumpLimit` option
Signed-off-by: Dwi Siswanto
* test(vardump): add test cases
Signed-off-by: Dwi Siswanto
* chore: tidy up mod
Signed-off-by: Dwi Siswanto
---------
Signed-off-by: Dwi Siswanto
---
cmd/nuclei/main.go | 1 +
go.mod | 3 +-
go.sum | 2 +
internal/runner/options.go | 1 +
pkg/protocols/code/code.go | 2 +-
.../helpers/eventcreator/eventcreator.go | 2 +-
pkg/protocols/common/utils/vardump/dump.go | 84 +++++++++++--------
.../common/utils/vardump/dump_test.go | 55 ++++++++++++
pkg/protocols/common/utils/vardump/vars.go | 8 ++
pkg/protocols/dns/request.go | 2 +-
pkg/protocols/headless/engine/page_actions.go | 2 +-
pkg/protocols/headless/request.go | 2 +-
pkg/protocols/http/build_request.go | 2 +-
pkg/protocols/javascript/js.go | 2 +-
pkg/protocols/network/request.go | 2 +-
pkg/protocols/ssl/ssl.go | 2 +-
pkg/protocols/websocket/websocket.go | 2 +-
pkg/protocols/whois/whois.go | 2 +-
pkg/types/types.go | 2 +
19 files changed, 131 insertions(+), 47 deletions(-)
create mode 100644 pkg/protocols/common/utils/vardump/dump_test.go
create mode 100644 pkg/protocols/common/utils/vardump/vars.go
diff --git a/cmd/nuclei/main.go b/cmd/nuclei/main.go
index caac7e5546..5e95a8b1b2 100644
--- a/cmd/nuclei/main.go
+++ b/cmd/nuclei/main.go
@@ -425,6 +425,7 @@ on extensive configurability, massive extensibility and ease of use.`)
flagSet.StringVar(&memProfile, "profile-mem", "", "generate memory (heap) profile & trace files"),
flagSet.BoolVar(&options.VerboseVerbose, "vv", false, "display templates loaded for scan"),
flagSet.BoolVarP(&options.ShowVarDump, "show-var-dump", "svd", false, "show variables dump for debugging"),
+ flagSet.IntVarP(&options.VarDumpLimit, "var-dump-limit", "vdl", 255, "limit the number of characters displayed in var dump"),
flagSet.BoolVarP(&options.EnablePprof, "enable-pprof", "ep", false, "enable pprof debugging server"),
flagSet.CallbackVarP(printTemplateVersion, "templates-version", "tv", "shows the version of the installed nuclei-templates"),
flagSet.BoolVarP(&options.HealthCheck, "health-check", "hc", false, "run diagnostic check up"),
diff --git a/go.mod b/go.mod
index 86a8a00643..db236f5908 100644
--- a/go.mod
+++ b/go.mod
@@ -1,6 +1,6 @@
module github.com/projectdiscovery/nuclei/v3
-go 1.21
+go 1.21.0
require (
github.com/Knetic/govaluate v3.0.1-0.20171022003610-9aa49832a739+incompatible
@@ -104,6 +104,7 @@ require (
github.com/shurcooL/graphql v0.0.0-20230722043721-ed46e5a46466
github.com/stretchr/testify v1.9.0
github.com/tarunKoyalwar/goleak v0.0.0-20240429141123-0efa90dbdcf9
+ github.com/yassinebenaid/godump v0.10.0
github.com/zmap/zgrab2 v0.1.8-0.20230806160807-97ba87c0e706
go.mongodb.org/mongo-driver v1.17.0
golang.org/x/term v0.24.0
diff --git a/go.sum b/go.sum
index b69d6283a5..556ba04f24 100644
--- a/go.sum
+++ b/go.sum
@@ -1104,6 +1104,8 @@ github.com/xhit/go-str2duration v1.2.0/go.mod h1:3cPSlfZlUHVlneIVfePFWcJZsuwf+P1
github.com/xi2/xz v0.0.0-20171230120015-48954b6210f8 h1:nIPpBwaJSVYIxUFsDv3M8ofmx9yWTog9BfvIu0q41lo=
github.com/xi2/xz v0.0.0-20171230120015-48954b6210f8/go.mod h1:HUYIGzjTL3rfEspMxjDjgmT5uz5wzYJKVo23qUhYTos=
github.com/xiang90/probing v0.0.0-20190116061207-43a291ad63a2/go.mod h1:UETIi67q53MR2AWcXfiuqkDkRtnGDLqkBTpCHuJHxtU=
+github.com/yassinebenaid/godump v0.10.0 h1:FolBA+Ix5uwUiXYBBYOsf1VkT5+0f4gtFNTkYTiIR08=
+github.com/yassinebenaid/godump v0.10.0/go.mod h1:dc/0w8wmg6kVIvNGAzbKH1Oa54dXQx8SNKh4dPRyW44=
github.com/yl2chen/cidranger v1.0.2 h1:lbOWZVCG1tCRX4u24kuM1Tb4nHqWkDxwLdoS+SevawU=
github.com/yl2chen/cidranger v1.0.2/go.mod h1:9U1yz7WPYDwf0vpNWFaeRh0bjwz5RVgRy/9UEQfHl0g=
github.com/youmark/pkcs8 v0.0.0-20240726163527-a2c0da244d78 h1:ilQV1hzziu+LLM3zUTJ0trRztfwgjqKnBWNtSRkbmwM=
diff --git a/internal/runner/options.go b/internal/runner/options.go
index 2872b96a7a..4ad62a855a 100644
--- a/internal/runner/options.go
+++ b/internal/runner/options.go
@@ -68,6 +68,7 @@ func ParseOptions(options *types.Options) {
if options.ShowVarDump {
vardump.EnableVarDump = true
+ vardump.Limit = options.VarDumpLimit
}
if options.ShowActions {
gologger.Info().Msgf("Showing available headless actions: ")
diff --git a/pkg/protocols/code/code.go b/pkg/protocols/code/code.go
index 87a0128645..4503b2117f 100644
--- a/pkg/protocols/code/code.go
+++ b/pkg/protocols/code/code.go
@@ -235,7 +235,7 @@ func (request *Request) ExecuteWithResults(input *contextargs.Context, dynamicVa
gologger.Verbose().Msgf("[%s] Executed code on local machine %v", request.options.TemplateID, input.MetaInput.Input)
if vardump.EnableVarDump {
- gologger.Debug().Msgf("Code Protocol request variables: \n%s\n", vardump.DumpVariables(allvars))
+ gologger.Debug().Msgf("Code Protocol request variables: %s\n", vardump.DumpVariables(allvars))
}
if request.options.Options.Debug || request.options.Options.DebugRequests {
diff --git a/pkg/protocols/common/helpers/eventcreator/eventcreator.go b/pkg/protocols/common/helpers/eventcreator/eventcreator.go
index 078963ceed..22c0a7810d 100644
--- a/pkg/protocols/common/helpers/eventcreator/eventcreator.go
+++ b/pkg/protocols/common/helpers/eventcreator/eventcreator.go
@@ -24,7 +24,7 @@ func CreateEventWithAdditionalOptions(request protocols.Request, outputEvent out
// Dump response variables if ran in debug mode
if vardump.EnableVarDump {
protoName := cases.Title(language.English).String(request.Type().String())
- gologger.Debug().Msgf("%v Protocol response variables: \n%s\n", protoName, vardump.DumpVariables(outputEvent))
+ gologger.Debug().Msgf("%v Protocol response variables: %s\n", protoName, vardump.DumpVariables(outputEvent))
}
for _, compiledOperator := range request.GetCompiledOperators() {
if compiledOperator != nil {
diff --git a/pkg/protocols/common/utils/vardump/dump.go b/pkg/protocols/common/utils/vardump/dump.go
index ab4f56b113..82de85e67b 100644
--- a/pkg/protocols/common/utils/vardump/dump.go
+++ b/pkg/protocols/common/utils/vardump/dump.go
@@ -1,53 +1,67 @@
package vardump
import (
- "strconv"
"strings"
"github.com/projectdiscovery/nuclei/v3/pkg/types"
mapsutil "github.com/projectdiscovery/utils/maps"
+ "github.com/yassinebenaid/godump"
)
-// EnableVarDump enables var dump for debugging optionally
-var EnableVarDump bool
+// variables is a map of variables
+type variables = map[string]any
-// DumpVariables writes the truncated dump of variables to a string
-// in a formatted key-value manner.
-//
-// The values are truncated to return 50 characters from start and end.
-func DumpVariables(data map[string]interface{}) string {
- var counter int
+// DumpVariables dumps the variables in a pretty format
+func DumpVariables(data variables) string {
+ if !EnableVarDump {
+ return ""
+ }
+
+ d := godump.Dumper{
+ Indentation: " ",
+ HidePrivateFields: false,
+ ShowPrimitiveNamedTypes: true,
+ }
+
+ d.Theme = godump.Theme{
+ String: godump.RGB{R: 138, G: 201, B: 38},
+ Quotes: godump.RGB{R: 112, G: 214, B: 255},
+ Bool: godump.RGB{R: 249, G: 87, B: 56},
+ Number: godump.RGB{R: 10, G: 178, B: 242},
+ Types: godump.RGB{R: 0, G: 150, B: 199},
+ Address: godump.RGB{R: 205, G: 93, B: 0},
+ PointerTag: godump.RGB{R: 110, G: 110, B: 110},
+ Nil: godump.RGB{R: 219, G: 57, B: 26},
+ Func: godump.RGB{R: 160, G: 90, B: 220},
+ Fields: godump.RGB{R: 189, G: 176, B: 194},
+ Chan: godump.RGB{R: 195, G: 154, B: 76},
+ UnsafePointer: godump.RGB{R: 89, G: 193, B: 180},
+ Braces: godump.RGB{R: 185, G: 86, B: 86},
+ }
- buffer := &strings.Builder{}
- buffer.Grow(len(data) * 78) // grow buffer to an approximate size
+ return d.Sprint(process(data, Limit))
+}
- builder := &strings.Builder{}
- // sort keys for deterministic output
+// process is a helper function that processes the variables
+// and returns a new map of variables
+func process(data variables, limit int) variables {
keys := mapsutil.GetSortedKeys(data)
+ vars := make(variables)
+
+ if limit == 0 {
+ limit = 255
+ }
for _, k := range keys {
- v := data[k]
- valueString := types.ToString(v)
-
- counter++
- if len(valueString) > 50 {
- builder.Grow(56)
- builder.WriteString(valueString[0:25])
- builder.WriteString(" .... ")
- builder.WriteString(valueString[len(valueString)-25:])
- valueString = builder.String()
- builder.Reset()
+ v := types.ToString(data[k])
+ v = strings.ReplaceAll(strings.ReplaceAll(v, "\r", " "), "\n", " ")
+ if len(v) > limit {
+ v = v[:limit]
+ v += " [...]"
}
- valueString = strings.ReplaceAll(strings.ReplaceAll(valueString, "\r", " "), "\n", " ")
-
- buffer.WriteString("\t")
- buffer.WriteString(strconv.Itoa(counter))
- buffer.WriteString(". ")
- buffer.WriteString(k)
- buffer.WriteString(" => ")
- buffer.WriteString(valueString)
- buffer.WriteString("\n")
+
+ vars[k] = v
}
- final := buffer.String()
- return final
+
+ return vars
}
diff --git a/pkg/protocols/common/utils/vardump/dump_test.go b/pkg/protocols/common/utils/vardump/dump_test.go
new file mode 100644
index 0000000000..9929fa5318
--- /dev/null
+++ b/pkg/protocols/common/utils/vardump/dump_test.go
@@ -0,0 +1,55 @@
+package vardump
+
+import (
+ "strings"
+ "testing"
+
+ "github.com/stretchr/testify/assert"
+)
+
+func TestDumpVariables(t *testing.T) {
+ // Enable var dump for testing
+ EnableVarDump = true
+
+ // Test case
+ testVars := variables{
+ "string": "test",
+ "int": 42,
+ "bool": true,
+ "slice": []string{"a", "b", "c"},
+ }
+
+ result := DumpVariables(testVars)
+
+ // Assertions
+ assert.NotEmpty(t, result)
+ assert.Contains(t, result, "string")
+ assert.Contains(t, result, "test")
+ assert.Contains(t, result, "int")
+ assert.Contains(t, result, "42")
+ assert.Contains(t, result, "bool")
+ assert.Contains(t, result, "true")
+ assert.Contains(t, result, "slice")
+ assert.Contains(t, result, "a")
+ assert.Contains(t, result, "b")
+ assert.Contains(t, result, "c")
+
+ // Test with EnableVarDump set to false
+ EnableVarDump = false
+ result = DumpVariables(testVars)
+ assert.Empty(t, result)
+}
+
+func TestProcess(t *testing.T) {
+ testVars := variables{
+ "short": "short string",
+ "long": strings.Repeat("a", 300),
+ "number": 42,
+ }
+
+ processed := process(testVars, 255)
+
+ assert.Equal(t, "short string", processed["short"])
+ assert.Equal(t, strings.Repeat("a", 255)+" [...]", processed["long"])
+ assert.Equal(t, "42", processed["number"])
+}
diff --git a/pkg/protocols/common/utils/vardump/vars.go b/pkg/protocols/common/utils/vardump/vars.go
new file mode 100644
index 0000000000..f5e18bce69
--- /dev/null
+++ b/pkg/protocols/common/utils/vardump/vars.go
@@ -0,0 +1,8 @@
+package vardump
+
+var (
+ // EnableVarDump enables var dump for debugging optionally
+ EnableVarDump bool
+ // Limit is the maximum characters to be dumped
+ Limit int = 255
+)
diff --git a/pkg/protocols/dns/request.go b/pkg/protocols/dns/request.go
index 9457845270..6e82c047bc 100644
--- a/pkg/protocols/dns/request.go
+++ b/pkg/protocols/dns/request.go
@@ -108,7 +108,7 @@ func (request *Request) ExecuteWithResults(input *contextargs.Context, metadata,
func (request *Request) execute(input *contextargs.Context, domain string, metadata, previous output.InternalEvent, vars map[string]interface{}, callback protocols.OutputEventCallback) error {
var err error
if vardump.EnableVarDump {
- gologger.Debug().Msgf("DNS Protocol request variables: \n%s\n", vardump.DumpVariables(vars))
+ gologger.Debug().Msgf("DNS Protocol request variables: %s\n", vardump.DumpVariables(vars))
}
// Compile each request for the template based on the URL
diff --git a/pkg/protocols/headless/engine/page_actions.go b/pkg/protocols/headless/engine/page_actions.go
index 18ed515662..640bb76214 100644
--- a/pkg/protocols/headless/engine/page_actions.go
+++ b/pkg/protocols/headless/engine/page_actions.go
@@ -334,7 +334,7 @@ func (p *Page) NavigateURL(action *Action, out ActionData, allvars map[string]in
allvars = generators.MergeMaps(allvars, defaultReqVars)
if vardump.EnableVarDump {
- gologger.Debug().Msgf("Headless Protocol request variables: \n%s\n", vardump.DumpVariables(allvars))
+ gologger.Debug().Msgf("Headless Protocol request variables: %s\n", vardump.DumpVariables(allvars))
}
// Evaluate the target url with all variables
diff --git a/pkg/protocols/headless/request.go b/pkg/protocols/headless/request.go
index 5f9b53174e..aae70aa347 100644
--- a/pkg/protocols/headless/request.go
+++ b/pkg/protocols/headless/request.go
@@ -122,7 +122,7 @@ func (request *Request) executeRequestWithPayloads(input *contextargs.Context, p
defer instance.Close()
if vardump.EnableVarDump {
- gologger.Debug().Msgf("Headless Protocol request variables: \n%s\n", vardump.DumpVariables(payloads))
+ gologger.Debug().Msgf("Headless Protocol request variables: %s\n", vardump.DumpVariables(payloads))
}
instance.SetInteractsh(request.options.Interactsh)
diff --git a/pkg/protocols/http/build_request.go b/pkg/protocols/http/build_request.go
index 1b046bffd8..bc3b41244e 100644
--- a/pkg/protocols/http/build_request.go
+++ b/pkg/protocols/http/build_request.go
@@ -204,7 +204,7 @@ func (r *requestGenerator) Make(ctx context.Context, input *contextargs.Context,
finalVars := generators.MergeMaps(allVars, payloads)
if vardump.EnableVarDump {
- gologger.Debug().Msgf("HTTP Protocol request variables: \n%s\n", vardump.DumpVariables(finalVars))
+ gologger.Debug().Msgf("HTTP Protocol request variables: %s\n", vardump.DumpVariables(finalVars))
}
// Note: If possible any changes to current logic (i.e evaluate -> then parse URL)
diff --git a/pkg/protocols/javascript/js.go b/pkg/protocols/javascript/js.go
index 0bd26b88a5..5953c9c296 100644
--- a/pkg/protocols/javascript/js.go
+++ b/pkg/protocols/javascript/js.go
@@ -319,7 +319,7 @@ func (request *Request) ExecuteWithResults(target *contextargs.Context, dynamicV
templateCtx.Merge(payloadValues)
if vardump.EnableVarDump {
- gologger.Debug().Msgf("Javascript Protocol request variables: \n%s\n", vardump.DumpVariables(payloadValues))
+ gologger.Debug().Msgf("JavaScript Protocol request variables: %s\n", vardump.DumpVariables(payloadValues))
}
if request.PreCondition != "" {
diff --git a/pkg/protocols/network/request.go b/pkg/protocols/network/request.go
index 32d4ae3494..29cdca4ad2 100644
--- a/pkg/protocols/network/request.go
+++ b/pkg/protocols/network/request.go
@@ -283,7 +283,7 @@ func (request *Request) executeRequestWithPayloads(variables map[string]interfac
interimValues := generators.MergeMaps(variables, payloads)
if vardump.EnableVarDump {
- gologger.Debug().Msgf("Network Protocol request variables: \n%s\n", vardump.DumpVariables(interimValues))
+ gologger.Debug().Msgf("Network Protocol request variables: %s\n", vardump.DumpVariables(interimValues))
}
inputEvents := make(map[string]interface{})
diff --git a/pkg/protocols/ssl/ssl.go b/pkg/protocols/ssl/ssl.go
index 554a4f5c10..fd0dae83db 100644
--- a/pkg/protocols/ssl/ssl.go
+++ b/pkg/protocols/ssl/ssl.go
@@ -222,7 +222,7 @@ func (request *Request) ExecuteWithResults(input *contextargs.Context, dynamicVa
payloadValues = generators.MergeMaps(variablesMap, payloadValues, request.options.Constants)
if vardump.EnableVarDump {
- gologger.Debug().Msgf("SSL Protocol request variables: \n%s\n", vardump.DumpVariables(payloadValues))
+ gologger.Debug().Msgf("SSL Protocol request variables: %s\n", vardump.DumpVariables(payloadValues))
}
finalAddress, dataErr := expressions.EvaluateByte([]byte(request.Address), payloadValues)
diff --git a/pkg/protocols/websocket/websocket.go b/pkg/protocols/websocket/websocket.go
index aa099ef43a..8eeeedf217 100644
--- a/pkg/protocols/websocket/websocket.go
+++ b/pkg/protocols/websocket/websocket.go
@@ -207,7 +207,7 @@ func (request *Request) executeRequestWithPayloads(target *contextargs.Context,
}
if vardump.EnableVarDump {
- gologger.Debug().Msgf("Websocket Protocol request variables: \n%s\n", vardump.DumpVariables(payloadValues))
+ gologger.Debug().Msgf("WebSocket Protocol request variables: %s\n", vardump.DumpVariables(payloadValues))
}
finalAddress, dataErr := expressions.EvaluateByte([]byte(request.Address), payloadValues)
diff --git a/pkg/protocols/whois/whois.go b/pkg/protocols/whois/whois.go
index 9963ec19f8..91d0edcf8a 100644
--- a/pkg/protocols/whois/whois.go
+++ b/pkg/protocols/whois/whois.go
@@ -99,7 +99,7 @@ func (request *Request) ExecuteWithResults(input *contextargs.Context, dynamicVa
variables := generators.MergeMaps(vars, defaultVars, optionVars, dynamicValues, request.options.Constants)
if vardump.EnableVarDump {
- gologger.Debug().Msgf("Whois Protocol request variables: \n%s\n", vardump.DumpVariables(variables))
+ gologger.Debug().Msgf("Whois Protocol request variables: %s\n", vardump.DumpVariables(variables))
}
// and replace placeholders
diff --git a/pkg/types/types.go b/pkg/types/types.go
index 9cc88f49ff..f6e7ab4470 100644
--- a/pkg/types/types.go
+++ b/pkg/types/types.go
@@ -206,6 +206,8 @@ type Options struct {
VerboseVerbose bool
// ShowVarDump displays variable dump
ShowVarDump bool
+ // VarDumpLimit limits the number of characters displayed in var dump
+ VarDumpLimit int
// No-Color disables the colored output.
NoColor bool
// UpdateTemplates updates the templates installed at startup (also used by cloud to update datasources)
From 98948d0266354fbbd7bbfbaf4041e7e8a59dcda7 Mon Sep 17 00:00:00 2001
From: Ramana Reddy <90540245+RamanaReddy0M@users.noreply.github.com>
Date: Mon, 14 Oct 2024 20:54:58 +0530
Subject: [PATCH 237/283] support stop-at-first-match for network templates
(#5554)
---
pkg/protocols/network/network.go | 4 ++++
pkg/protocols/network/request.go | 33 ++++++++++++++++++++++++++++----
2 files changed, 33 insertions(+), 4 deletions(-)
diff --git a/pkg/protocols/network/network.go b/pkg/protocols/network/network.go
index 70d618dcb5..c90f5e7019 100644
--- a/pkg/protocols/network/network.go
+++ b/pkg/protocols/network/network.go
@@ -85,6 +85,10 @@ type Request struct {
// SelfContained specifies if the request is self-contained.
SelfContained bool `yaml:"-" json:"-"`
+ // description: |
+ // StopAtFirstMatch stops the execution of the requests and template as soon as a match is found.
+ StopAtFirstMatch bool `yaml:"stop-at-first-match,omitempty" json:"stop-at-first-match,omitempty" jsonschema:"title=stop at first match,description=Stop the execution after a match is found"`
+
// description: |
// ports is post processed list of ports to scan (obtained from Port)
ports []string `yaml:"-" json:"-"`
diff --git a/pkg/protocols/network/request.go b/pkg/protocols/network/request.go
index 29cdca4ad2..10c71b04db 100644
--- a/pkg/protocols/network/request.go
+++ b/pkg/protocols/network/request.go
@@ -8,6 +8,7 @@ import (
"os"
"strings"
"sync"
+ "sync/atomic"
"time"
"github.com/pkg/errors"
@@ -99,6 +100,16 @@ func (request *Request) ExecuteWithResults(target *contextargs.Context, metadata
gologger.Verbose().Msgf("[%v] got errors while checking open ports: %s\n", request.options.TemplateID, err)
}
+ // stop at first match if requested
+ atomicBool := &atomic.Bool{}
+ shouldStopAtFirstMatch := request.StopAtFirstMatch || request.options.StopAtFirstMatch || request.options.Options.StopAtFirstMatch
+ wrappedCallback := func(event *output.InternalWrappedEvent) {
+ if event != nil && event.HasOperatorResult() {
+ atomicBool.Store(true)
+ }
+ callback(event)
+ }
+
for _, port := range ports {
input := target.Clone()
// use network port updates input with new port requested in template file
@@ -107,9 +118,12 @@ func (request *Request) ExecuteWithResults(target *contextargs.Context, metadata
if err := input.UseNetworkPort(port, request.ExcludePorts); err != nil {
gologger.Debug().Msgf("Could not network port from constants: %s\n", err)
}
- if err := request.executeOnTarget(input, visitedAddresses, metadata, previous, callback); err != nil {
+ if err := request.executeOnTarget(input, visitedAddresses, metadata, previous, wrappedCallback); err != nil {
return err
}
+ if shouldStopAtFirstMatch && atomicBool.Load() {
+ break
+ }
}
return nil
@@ -141,6 +155,16 @@ func (request *Request) executeOnTarget(input *contextargs.Context, visited maps
variablesMap := request.options.Variables.Evaluate(variables)
variables = generators.MergeMaps(variablesMap, variables, request.options.Constants)
+ // stop at first match if requested
+ atomicBool := &atomic.Bool{}
+ shouldStopAtFirstMatch := request.StopAtFirstMatch || request.options.StopAtFirstMatch || request.options.Options.StopAtFirstMatch
+ wrappedCallback := func(event *output.InternalWrappedEvent) {
+ if event != nil && event.HasOperatorResult() {
+ atomicBool.Store(true)
+ }
+ callback(event)
+ }
+
for _, kv := range request.addresses {
select {
case <-input.Context().Done():
@@ -154,12 +178,13 @@ func (request *Request) executeOnTarget(input *contextargs.Context, visited maps
continue
}
visited.Set(actualAddress, struct{}{})
-
- if err = request.executeAddress(variables, actualAddress, address, input, kv.tls, previous, callback); err != nil {
+ if err = request.executeAddress(variables, actualAddress, address, input, kv.tls, previous, wrappedCallback); err != nil {
outputEvent := request.responseToDSLMap("", "", "", address, "")
callback(&output.InternalWrappedEvent{InternalEvent: outputEvent})
gologger.Warning().Msgf("[%v] Could not make network request for (%s) : %s\n", request.options.TemplateID, actualAddress, err)
- continue
+ }
+ if shouldStopAtFirstMatch && atomicBool.Load() {
+ break
}
}
return err
From 159a8a53cfd258e4c2556e6d24bc63957e78e1ed Mon Sep 17 00:00:00 2001
From: ghost
Date: Mon, 14 Oct 2024 15:26:08 +0000
Subject: [PATCH 238/283] Auto Generate Syntax Docs + JSONSchema [Mon Oct 14
15:26:08 UTC 2024] :robot:
---
SYNTAX-REFERENCE.md | 13 +++++++++++++
nuclei-jsonschema.json | 5 +++++
pkg/templates/templates_doc.go | 7 ++++++-
3 files changed, 24 insertions(+), 1 deletion(-)
diff --git a/SYNTAX-REFERENCE.md b/SYNTAX-REFERENCE.md
index c448d251d3..6d1d9effc4 100755
--- a/SYNTAX-REFERENCE.md
+++ b/SYNTAX-REFERENCE.md
@@ -3223,6 +3223,19 @@ read-all: false
+
+
+stop-at-first-match
bool
+
+
+
+
+StopAtFirstMatch stops the execution of the requests and template as soon as a match is found.
+
+
+
+
+
diff --git a/nuclei-jsonschema.json b/nuclei-jsonschema.json
index 8a8d5dd1ba..97477e1f7e 100644
--- a/nuclei-jsonschema.json
+++ b/nuclei-jsonschema.json
@@ -1370,6 +1370,11 @@
"title": "read all response stream",
"description": "Read all response stream till the server stops sending"
},
+ "stop-at-first-match": {
+ "type": "boolean",
+ "title": "stop at first match",
+ "description": "Stop the execution after a match is found"
+ },
"matchers": {
"items": {
"$ref": "#/$defs/matchers.Matcher"
diff --git a/pkg/templates/templates_doc.go b/pkg/templates/templates_doc.go
index 9a7fc97a37..9bf754e24c 100644
--- a/pkg/templates/templates_doc.go
+++ b/pkg/templates/templates_doc.go
@@ -1334,7 +1334,7 @@ func init() {
Value: "Full Network protocol data",
},
}
- NETWORKRequestDoc.Fields = make([]encoder.Doc, 10)
+ NETWORKRequestDoc.Fields = make([]encoder.Doc, 11)
NETWORKRequestDoc.Fields[0].Name = "id"
NETWORKRequestDoc.Fields[0].Type = "string"
NETWORKRequestDoc.Fields[0].Note = ""
@@ -1393,6 +1393,11 @@ func init() {
NETWORKRequestDoc.Fields[9].Comments[encoder.LineComment] = "ReadAll determines if the data stream should be read till the end regardless of the size"
NETWORKRequestDoc.Fields[9].AddExample("", false)
+ NETWORKRequestDoc.Fields[10].Name = "stop-at-first-match"
+ NETWORKRequestDoc.Fields[10].Type = "bool"
+ NETWORKRequestDoc.Fields[10].Note = ""
+ NETWORKRequestDoc.Fields[10].Description = "StopAtFirstMatch stops the execution of the requests and template as soon as a match is found."
+ NETWORKRequestDoc.Fields[10].Comments[encoder.LineComment] = "StopAtFirstMatch stops the execution of the requests and template as soon as a match is found."
NETWORKInputDoc.Type = "network.Input"
NETWORKInputDoc.Comments[encoder.LineComment] = ""
From 7e4b4a8c55f2ef1c3495f8c41cfea9d77dd7b7df Mon Sep 17 00:00:00 2001
From: Ramana Reddy <90540245+RamanaReddy0M@users.noreply.github.com>
Date: Fri, 18 Oct 2024 20:44:40 +0530
Subject: [PATCH 239/283] fix: interactsh-url placeholder replacement in
variables for network template (#5677)
---
pkg/protocols/network/request.go | 25 +++++++++++++------------
1 file changed, 13 insertions(+), 12 deletions(-)
diff --git a/pkg/protocols/network/request.go b/pkg/protocols/network/request.go
index 10c71b04db..3579acd3b8 100644
--- a/pkg/protocols/network/request.go
+++ b/pkg/protocols/network/request.go
@@ -314,30 +314,31 @@ func (request *Request) executeRequestWithPayloads(variables map[string]interfac
inputEvents := make(map[string]interface{})
for _, input := range request.Inputs {
- data := []byte(input.Data)
+ dataInBytes := []byte(input.Data)
+ var err error
- if request.options.Interactsh != nil {
- var transformedData string
- transformedData, interactshURLs = request.options.Interactsh.Replace(string(data), []string{})
- data = []byte(transformedData)
- }
-
- finalData, err := expressions.EvaluateByte(data, interimValues)
+ dataInBytes, err = expressions.EvaluateByte(dataInBytes, interimValues)
if err != nil {
request.options.Output.Request(request.options.TemplatePath, address, request.Type().String(), err)
request.options.Progress.IncrementFailedRequestsBy(1)
return errors.Wrap(err, "could not evaluate template expressions")
}
- reqBuilder.Write(finalData)
+ data := string(dataInBytes)
+ if request.options.Interactsh != nil {
+ data, interactshURLs = request.options.Interactsh.Replace(data, []string{})
+ dataInBytes = []byte(data)
+ }
+
+ reqBuilder.Write(dataInBytes)
- if err := expressions.ContainsUnresolvedVariables(string(finalData)); err != nil {
+ if err := expressions.ContainsUnresolvedVariables(data); err != nil {
gologger.Warning().Msgf("[%s] Could not make network request for %s: %v\n", request.options.TemplateID, actualAddress, err)
return nil
}
if input.Type.GetType() == hexType {
- finalData, err = hex.DecodeString(string(finalData))
+ dataInBytes, err = hex.DecodeString(data)
if err != nil {
request.options.Output.Request(request.options.TemplatePath, address, request.Type().String(), err)
request.options.Progress.IncrementFailedRequestsBy(1)
@@ -345,7 +346,7 @@ func (request *Request) executeRequestWithPayloads(variables map[string]interfac
}
}
- if _, err := conn.Write(finalData); err != nil {
+ if _, err := conn.Write(dataInBytes); err != nil {
request.options.Output.Request(request.options.TemplatePath, address, request.Type().String(), err)
request.options.Progress.IncrementFailedRequestsBy(1)
return errors.Wrap(err, "could not write request to server")
From d7c8c8af8007a5e6175cc51322bd7a9d50ec7706 Mon Sep 17 00:00:00 2001
From: sandeep <8293321+ehsandeep@users.noreply.github.com>
Date: Fri, 18 Oct 2024 22:28:02 +0530
Subject: [PATCH 240/283] version update
---
pkg/catalog/config/constants.go | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/pkg/catalog/config/constants.go b/pkg/catalog/config/constants.go
index e29af00cdd..7d11b6f71e 100644
--- a/pkg/catalog/config/constants.go
+++ b/pkg/catalog/config/constants.go
@@ -31,7 +31,7 @@ const (
CLIConfigFileName = "config.yaml"
ReportingConfigFilename = "reporting-config.yaml"
// Version is the current version of nuclei
- Version = `v3.3.4`
+ Version = `v3.3.5`
// Directory Names of custom templates
CustomS3TemplatesDirName = "s3"
CustomGitHubTemplatesDirName = "github"
From 44f398c08da62acbf555d3b21f7260e5daff2ce1 Mon Sep 17 00:00:00 2001
From: sandeep <8293321+ehsandeep@users.noreply.github.com>
Date: Sat, 19 Oct 2024 17:21:13 +0530
Subject: [PATCH 241/283] readme updates
---
README.md | 372 +++++++++++++------
static/nuclei-cover-image.png | Bin 0 -> 818033 bytes
static/nuclei-cover.png | Bin 0 -> 818033 bytes
static/nuclei-getting-started.png | Bin 0 -> 45304 bytes
static/nuclei-template-example.png | Bin 0 -> 2155430 bytes
static/nuclei-templates-teamcity-example.png | Bin 0 -> 1162315 bytes
static/nuclei-templates-teamcity.png | Bin 0 -> 1538352 bytes
static/nuclei-write-your-first-template.png | Bin 0 -> 47472 bytes
static/projectdiscovery-browse-results.gif | Bin 0 -> 2862660 bytes
static/teamcity-example.png | Bin 0 -> 1098783 bytes
10 files changed, 252 insertions(+), 120 deletions(-)
create mode 100644 static/nuclei-cover-image.png
create mode 100644 static/nuclei-cover.png
create mode 100644 static/nuclei-getting-started.png
create mode 100644 static/nuclei-template-example.png
create mode 100644 static/nuclei-templates-teamcity-example.png
create mode 100644 static/nuclei-templates-teamcity.png
create mode 100644 static/nuclei-write-your-first-template.png
create mode 100644 static/projectdiscovery-browse-results.gif
create mode 100644 static/teamcity-example.png
diff --git a/README.md b/README.md
index 412d348645..02cacb9ccf 100644
--- a/README.md
+++ b/README.md
@@ -1,30 +1,4 @@
-
-
-
-
-
-Fast and customisable vulnerability scanner based on simple YAML based DSL.
-
-
-
-
-
-
-
-
-
-
-
-
-
-
- How •
- Install •
- Documentation •
- Credits •
- FAQs •
- Join Discord
-
+
English •
@@ -34,78 +8,103 @@
Spanish •
日本語
-
----
-
-Nuclei is used to send requests across targets based on a template, leading to zero false positives and providing fast scanning on a large number of hosts. Nuclei offers scanning for a variety of protocols, including TCP, DNS, HTTP, SSL, File, Whois, Websocket, Headless, Code etc. With powerful and flexible templating, Nuclei can be used to model all kinds of security checks.
+
-We have a [dedicated repository](https://github.com/projectdiscovery/nuclei-templates) that houses various type of vulnerability templates contributed by **more than 300** security researchers and engineers.
+
-## How it works
+
+
+
+
+
+
+
+---
-
-
-
+
+
+
+
+Nuclei is a modern, high-performance vulnerability scanner that leverages simple YAML-based templates. It empowers you to design custom vulnerability detection scenarios that mimic real-world conditions, leading to zero false positives.
+
+- Simple YAML format for creating and customizing vulnerability templates.
+- Contributed by thousands of security professionals to tackle trending vulnerabilities.
+- Reduce false positives by simulating real-world steps to verify a vulnerability.
+- Ultra-fast parallel scan processing and request clustering.
+- Integrate into CI/CD pipelines for vulnerability detection and regression testing.
+- Supports multiple protocols like TCP, DNS, HTTP, SSL, WHOIS JavaScript, Code and more.
+- Integrate with Jira, Splunk, GitHub, Elastic, GitLab.
+
+## Table of Contents
+
+- [Get Started](#get-started)
+ - [1. Nuclei CLI](#1-nuclei-cli)
+ - [2. Pro and Enterprise Editions](#2-pro-and-enterprise-editions)
+- [Documentation](#documentation)
+ - [Command Line Flags](#command-line-flags)
+ - [Single target scan](#single-target-scan)
+ - [Scanning multiple targets](#scanning-multiple-targets)
+ - [Network scan](#network-scan)
+ - [Scanning with your custom template](#scanning-with-your-custom-template)
+ - [Connect Nuclei to ProjectDiscovery](#connect-nuclei-to-projectdiscovery)
+- [Nuclei Templates, Community and Rewards 💎](#nuclei-templates-community-and-rewards-)
+- [Our Mission](#our-mission)
+- [Contributors ❤️](#contributors-️)
+- [License](#license)
+---
| :exclamation: **Disclaimer** |
|---------------------------------|
| **This project is in active development**. Expect breaking changes with releases. Review the release changelog before updating. |
-| This project was primarily built to be used as a standalone CLI tool. **Running nuclei as a service may pose security risks.** It's recommended to use with caution and additional security measures. |
+| This project is primarily built to be used as a standalone CLI tool. **Running nuclei as a service may pose security risks.** It's recommended to use with caution and additional security measures. |
-# Install Nuclei
+## Get Started
-Nuclei requires **go1.21** to install successfully. Run the following command to install the latest version -
+### **1. Nuclei CLI**
-```sh
-go install -v github.com/projectdiscovery/nuclei/v3/cmd/nuclei@latest
-```
+Install Nuclei on your machine. Get started by following the installation guide [here](https://docs.projectdiscovery.io/tools/nuclei/install?utm_source=github&utm_medium=web&utm_campaign=nuclei_readme). Additionally, We provide [a free cloud tier](https://cloud.projectdiscovery.io/sign-up) and comes with a generous monthly free limits:
-
- Brew
-
- ```sh
- brew install nuclei
- ```
-
-
-
- Docker
-
- ```sh
- docker pull projectdiscovery/nuclei:latest
- ```
-
-
+- Store and visualize your vulnerability findings
+- Write and manage your nuclei templates
+- Access latest nuclei templates
+- Discover and store your targets
+
+### **2. Pro and Enterprise Editions**
-**More installation [methods can be found here](https://docs.projectdiscovery.io/tools/nuclei/install).**
+For security teams and enterprises, we provide a cloud-hosted service built on top of Nuclei OSS, fine-tuned to help you continuously run vulnerability scans at scale with your team and existing workflows:
-
-
-
+- 50x faster scans
+- Large scale scanning with high accuracy
+- Integrations with cloud services (AWS, GCP, Azure, CloudFlare, Fastly, Terraform, Kubernetes)
+- Jira, Slack, Linear, APIs and Webhooks
+- Executive and compliance reporting
+- Plus: Real-time scanning, SAML SSO, SOC 2 compliant platform (with EU and US hosting options), shared team workspaces, and more
+- We're constantly [adding new features](https://feedback.projectdiscovery.io/changelog)!
+- **Ideal for:** Pentesters, security teams, and enterprises
-### Nuclei Templates
+## Documentation
-Nuclei has built-in support for automatic template download/update as default since version [v2.5.2](https://github.com/projectdiscovery/nuclei/releases/tag/v2.5.2). [**Nuclei-Templates**](https://github.com/projectdiscovery/nuclei-templates) project provides a community-contributed list of ready-to-use templates that is constantly updated.
+Browse the full Nuclei [documentation here](https://docs.projectdiscovery.io/tools/nuclei/running). If you’re new to Nuclei, check out our [foundational Youtube series.](https://www.youtube.com/playlist?list=PLZRbR9aMzTTpItEdeNSulo8bYsvil80Rl)
-You may still use the `update-templates` flag to update the nuclei templates at any time; You can write your own checks for your individual workflow and needs following Nuclei's [templating guide](https://docs.projectdiscovery.io/templates/).
+
-The YAML DSL reference syntax is available [here](SYNTAX-REFERENCE.md).
+
- |
-
-
+
-### Usage
+### Command Line Flags
+
+To display all the flags for the tool:
```sh
nuclei -h
```
-This will display help for the tool. Here are all the switches it supports.
-
+
+ Expand full help flags
```console
Nuclei is a fast, template based vulnerability scanner focusing
@@ -279,23 +278,24 @@ HEADLESS:
-lha, -list-headless-action list available headless actions
DEBUG:
- -debug show all requests and responses
- -dreq, -debug-req show all sent requests
- -dresp, -debug-resp show all received responses
- -p, -proxy string[] list of http/socks5 proxy to use (comma separated or file input)
- -pi, -proxy-internal proxy all internal requests
- -ldf, -list-dsl-function list all supported DSL function signatures
- -tlog, -trace-log string file to write sent requests trace log
- -elog, -error-log string file to write sent requests error log
- -version show nuclei version
- -hm, -hang-monitor enable nuclei hang monitoring
- -v, -verbose show verbose output
- -profile-mem string optional nuclei memory profile dump file
- -vv display templates loaded for scan
- -svd, -show-var-dump show variables dump for debugging
- -ep, -enable-pprof enable pprof debugging server
- -tv, -templates-version shows the version of the installed nuclei-templates
- -hc, -health-check run diagnostic check up
+ -debug show all requests and responses
+ -dreq, -debug-req show all sent requests
+ -dresp, -debug-resp show all received responses
+ -p, -proxy string[] list of http/socks5 proxy to use (comma separated or file input)
+ -pi, -proxy-internal proxy all internal requests
+ -ldf, -list-dsl-function list all supported DSL function signatures
+ -tlog, -trace-log string file to write sent requests trace log
+ -elog, -error-log string file to write sent requests error log
+ -version show nuclei version
+ -hm, -hang-monitor enable nuclei hang monitoring
+ -v, -verbose show verbose output
+ -profile-mem string generate memory (heap) profile & trace files
+ -vv display templates loaded for scan
+ -svd, -show-var-dump show variables dump for debugging
+ -vdl, -var-dump-limit int limit the number of characters displayed in var dump (default 255)
+ -ep, -enable-pprof enable pprof debugging server
+ -tv, -templates-version shows the version of the installed nuclei-templates
+ -hc, -health-check run diagnostic check up
UPDATE:
-up, -update update nuclei engine to the latest released version
@@ -310,11 +310,13 @@ STATISTICS:
-mp, -metrics-port int port to expose nuclei metrics on (default 9092)
CLOUD:
- -auth configure projectdiscovery cloud (pdcp) api key (default true)
- -tid, -team-id string upload scan results to given team id (optional) (default "none")
- -cup, -cloud-upload upload scan results to pdcp dashboard
- -sid, -scan-id string upload scan results to existing scan id (optional)
- -sname, -scan-name string scan name to set (optional)
+ -auth configure projectdiscovery cloud (pdcp) api key (default true)
+ -tid, -team-id string upload scan results to given team id (optional) (default "none")
+ -cup, -cloud-upload upload scan results to pdcp dashboard [DEPRECATED use -dashboard]
+ -sid, -scan-id string upload scan results to existing scan id (optional)
+ -sname, -scan-name string scan name to set (optional)
+ -pd, -dashboard upload / view nuclei results in projectdiscovery cloud (pdcp) UI dashboard
+ -pdu, -dashboard-upload string upload / view nuclei results file (jsonl) in projectdiscovery cloud (pdcp) UI dashboard
AUTHENTICATION:
-sf, -secret-file string[] path to config file containing secrets for nuclei authenticated scan
@@ -323,59 +325,189 @@ AUTHENTICATION:
EXAMPLES:
Run nuclei on single host:
- $ nuclei -target example.com
+ $ nuclei -target example.com
Run nuclei with specific template directories:
- $ nuclei -target example.com -t http/cves/ -t ssl
+ $ nuclei -target example.com -t http/cves/ -t ssl
Run nuclei against a list of hosts:
- $ nuclei -list hosts.txt
+ $ nuclei -list hosts.txt
Run nuclei with a JSON output:
- $ nuclei -target example.com -json-export output.json
+ $ nuclei -target example.com -json-export output.json
Run nuclei with sorted Markdown outputs (with environment variables):
- $ MARKDOWN_EXPORT_SORT_MODE=template nuclei -target example.com -markdown-export nuclei_report/
+ $ MARKDOWN_EXPORT_SORT_MODE=template nuclei -target example.com -markdown-export nuclei_report/
Additional documentation is available at: https://docs.nuclei.sh/getting-started/running
```
-### Running Nuclei
+Additional documentation is available at: [https://docs.nuclei.sh/getting-started/running](https://docs.nuclei.sh/getting-started/running?utm_source=github&utm_medium=web&utm_campaign=nuclei_readme)
-See https://docs.projectdiscovery.io/tools/nuclei/running for details on running Nuclei
+
-### Using Nuclei From Go Code
+### Single target scan
-Complete guide of using Nuclei as Library/SDK is available at [godoc](https://pkg.go.dev/github.com/projectdiscovery/nuclei/v3/lib#section-readme)
+To perform a quick scan on web-application:
+```sh
+nuclei -target https://example.com
+```
-### Resources
+### Scanning multiple targets
-You can access the main documentation for Nuclei at https://docs.projectdiscovery.io/tools/nuclei/, and learn more about Nuclei in the cloud with [ProjectDiscovery Cloud Platform](https://cloud.projectdiscovery.io)
+Nuclei can handle bulk scanning by providing a list of targets. You can use a file containing multiple URLs.
-See https://docs.projectdiscovery.io/tools/nuclei/resources for more resources and videos about Nuclei!
+```sh
+nuclei -targets urls.txt
+```
-### Credits
+### Network scan
-Thanks to all the amazing [community contributors for sending PRs](https://github.com/projectdiscovery/nuclei/graphs/contributors) and keeping this project updated. :heart:
+This will scan the entire subnet for network-related issues, such as open ports or misconfigured services.
+
+```sh
+nuclei -target 192.168.1.0/24
+```
-If you have an idea or some kind of improvement, you are welcome to contribute and participate in the Project, feel free to send your PR.
+### Scanning with your custom template
-
-
-
-
+To write and use your own template, create a `.yaml` file with specific rules, then use it as follows.
+
+```sh
+nuclei -u https://example.com -t /path/to/your-template.yaml
+```
+
+### Connect Nuclei to ProjectDiscovery
+
+You can run the scans on your machine and upload the results to the cloud platform for further analysis and remediation.
+
+```sh
+nuclei -target https://example.com -dashboard
+```
+
+> [!NOTE]
+> This feature is absolutely free and does not require any subscription. For a detailed guide, refer to the [documentation](https://docs.projectdiscovery.io/cloud/scanning/nuclei-scan?utm_source=github&utm_medium=web&utm_campaign=nuclei_readme).
+
+## Nuclei Templates, Community and Rewards 💎
+[Nuclei templates](https://github.com/projectdiscovery/nuclei-templates) are based on the concepts of YAML based template files that define how the requests will be sent and processed. This allows easy extensibility capabilities to nuclei. The templates are written in YAML which specifies a simple human-readable format to quickly define the execution process.
+
+Try it online with our free AI powered Nuclei Templates Editor by [clicking here.](https://cloud.projectdiscovery.io/templates)
+
+Nuclei Templates offer a streamlined way to identify and communicate vulnerabilities, combining essential details like severity ratings and detection methods. This open-source, community-developed tool accelerates threat response and is widely recognized in the cybersecurity world. Nuclei templates are actively contributed by thousands of security researchers globally. We run two programs for our contributors: [Pioneers](https://projectdiscovery.io/pioneers) and [💎 bounties](https://github.com/projectdiscovery/nuclei-templates/issues?q=is%3Aissue%20state%3Aopen%20label%3A%22%F0%9F%92%8E%20Bounty%22).
+
+
+
+
+#### Examples
+
+Visit [our documentation](https://docs.projectdiscovery.io/templates/introduction) for use cases and ideas.
+
+| Use case | Nuclei template |
+| :----------------------------------- | :------------------------------------------------- |
+| Detect known CVEs | **[CVE-2021-44228 (Log4Shell)](https://cloud.projectdiscovery.io/public/CVE-2021-45046)** |
+| Identify Out-of-Band vulnerabilities | **[Blind SQL Injection via OOB](https://cloud.projectdiscovery.io/public/CVE-2024-22120)** |
+| SQL Injection detection | **[Generic SQL Injection](https://cloud.projectdiscovery.io/public/CVE-2022-34265)** |
+| Cross-Site Scripting (XSS) | **[Reflected XSS Detection](https://cloud.projectdiscovery.io/public/CVE-2023-4173)** |
+| Default or weak passwords | **[Default Credentials Check](https://cloud.projectdiscovery.io/public/airflow-default-login)** |
+| Secret files or data exposure | **[Sensitive File Disclosure](https://cloud.projectdiscovery.io/public/airflow-configuration-exposure)** |
+| Identify open redirects | **[Open Redirect Detection](https://cloud.projectdiscovery.io/public/open-redirect)** |
+| Detect subdomain takeovers | **[Subdomain Takeover Templates](https://cloud.projectdiscovery.io/public/azure-takeover-detection)** |
+| Security misconfigurations | **[Unprotected Jenkins Console](https://cloud.projectdiscovery.io/public/unauthenticated-jenkins)** |
+| Weak SSL/TLS configurations | **[SSL Certificate Expiry](https://cloud.projectdiscovery.io/public/expired-ssl)** |
+| Misconfigured cloud services | **[Open S3 Bucket Detection](https://cloud.projectdiscovery.io/public/s3-public-read-acp)** |
+| Remote code execution vulnerabilities| **[RCE Detection Templates](https://cloud.projectdiscovery.io/public/CVE-2024-29824)** |
+| Directory traversal attacks | **[Path Traversal Detection](https://cloud.projectdiscovery.io/public/oracle-fatwire-lfi)** |
+| File inclusion vulnerabilities | **[Local/Remote File Inclusion](https://cloud.projectdiscovery.io/public/CVE-2023-6977)** |
+
-Do also check out the below similar open-source projects that may fit in your workflow:
+## Our Mission
-[FFuF](https://github.com/ffuf/ffuf), [Qsfuzz](https://github.com/ameenmaali/qsfuzz), [Inception](https://github.com/proabiral/inception), [Snallygaster](https://github.com/hannob/snallygaster), [Gofingerprint](https://github.com/Static-Flow/gofingerprint), [Sn1per](https://github.com/1N3/Sn1per/tree/master/templates), [Google tsunami](https://github.com/google/tsunami-security-scanner), [Jaeles](https://github.com/jaeles-project/jaeles), [ChopChop](https://github.com/michelin/ChopChop)
+Traditional vulnerability scanners were built decades ago. They are closed-source, incredibly slow, and vendor-driven. Today's attackers are mass exploiting newly released CVEs across the internet within days, unlike the years it used to take. This shift requires a completely different approach to tackling trending exploits on the internet.
+
+We built Nuclei to solve this challenge. We made the entire scanning engine framework open and customizable—allowing the global security community to collaborate and tackle the trending attack vectors and vulnerabilities on the internet. Nuclei is now used and contributed by Fortune 500 enterprises, government agencies, universities.
+
+You can participate by contributing to our code, [templates library](https://github.com/projectdiscovery/nuclei-templates), or [joining our team.](https://projectdiscovery.io/)
+
+## Contributors :heart:
+
+Thanks to all the amazing [community contributors for sending PRs](https://github.com/projectdiscovery/nuclei/graphs/contributors) and keeping this project updated. :heart:
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
-### License
+## License
-Nuclei is distributed under [MIT License](https://github.com/projectdiscovery/nuclei/blob/main/LICENSE.md)
+Nuclei is distributed under [MIT License](https://github.com/projectdiscovery/nuclei/blob/main/LICENSE.md).
-
-
-
+
diff --git a/static/nuclei-cover-image.png b/static/nuclei-cover-image.png
new file mode 100644
index 0000000000000000000000000000000000000000..caab6fb2d4ec9d5f02b30220f62d5bb3e3bfd9ce
GIT binary patch
literal 818033
zcmWh!dpr~B|F7djox(IpsbZ
z!br?*hIOzF!`R4uW6Z{W{r-4f&+GZTo`0U#>v=x!&-?v(KdHBz9prye{pG-c1M-g7
zuKs=Cz>(Mk2YxaEcYQZl4QcNjz-jn%yd;N+^lZN!I
z!iY9l@{`@+PeZ^~EPf(XZA(>!;KhQ>U$c3y!Fm?aoxG3&E+#&;+rFPQQ?UV1)iF^P
zTg)>hPj>OB%SvL%__vtp^QhO*@7bGl|L2_{^@GMpLKahFM7ya~q#$}88}ND5XZc;mo#%8*gwc!WBeY!I`B%-|bB316$fg0jcTtALl-r2G4lx4d+94k=5gn!r#_B-4_Dt8BVRUBlR|&F
zQHohH7an#>M+-Hx#2+H|@i!fh2Hye`4~=vhm4rc0H9N&)P)ieFMt1`@F~n#R|7xF*
zkpp3&$SVfvW(YXjh$eAS$w5yhBg
z4I*58Xi=E|Z6j&pHCXh&_%P3{ovb}I#f8u5+S$c>;6{JFK5@1#ujAW_vgSxG$M1V$
zRMJMSsigvGf3z0aHHMNV{G)z;EuP!Pa%Rk4Px?Hcrd(PiM=~E<-yto+H~tGNC*nEM
zIs`#t)>U^e6V5q068xTqQCJ<7^ZKX_YC)|yBeAWXi?6Mj$$+X#vUhQiX9ton@t*T<
z>t0pPrhxf$rXKlyfdcX(MhgNxnuNd=XFYgx@cuc^?igOSmAUKKtdgi^@TRtRy=nep
zea6YC)?qjUrmQVJf2_ZHv9&jkv-TB~nWU;lJWh{s7c}VU;qVE&_R|m>6$;qyjDBXR
zu|Ln!byT)I@@8HZ<+b3HYrVrunukpJeY%_D8Vew*Ru*9NhW_6)!@Nn&l#!ZTw`y-dIAb
zgeH~H?zr7#Nq4fOJMMPuP@7E&FJYh8pJlhd!|yjB$4dM@rymK7cxyr!;H7B9@4N{O
zqWXWw-(4Rp+#j-LyFl173E#Q<60U0K9I(mX2^sI8C@79QM=9InHBZs(q>_%Y>{r9<
zVmj0=ZaPcBS~$N`xODKXS-8_F3p|7!feN(Hc|VV+suz_6zj79j=qQjRb7n9x6y|j{
zt0>7h^QEkyp=-Sz8NX`n?M8^j=D#E&6{coGcClGnntk-(ny$`ITkA|?LHr?38P7kE
z7q5ONvgBbr^0QmNp+esi<1>nLuZ&`U+43$nYqBgSHCQk;d>eargL~FkOmJW$2V1G%
z_({93bbeNr9((KLDqMmW5d0R_{7+$OrZ^`%Cl;75W@=g~w?}ou*vpl}Gr_5>d@!lj
zU>wg