Skip to content

Commit

Permalink
add -store-field-dir flag (#877)
Browse files Browse the repository at this point in the history
  • Loading branch information
dogancanbakir committed Jun 27, 2024
1 parent d835f94 commit 5ae80ae
Show file tree
Hide file tree
Showing 7 changed files with 16 additions and 7 deletions.
3 changes: 2 additions & 1 deletion README.md
Original file line number Diff line number Diff line change
Expand Up @@ -193,6 +193,7 @@ OUTPUT:
-o, -output string file to write output to
-sr, -store-response store http requests/responses
-srd, -store-response-dir string store http requests/responses to custom directory
-sfd, -store-field-dir string store per-host field to custom directory
-or, -omit-raw omit raw requests/responses from jsonl output
-ob, -omit-body omit response body from jsonl output
-j, -jsonl write output in jsonl format
Expand Down Expand Up @@ -688,7 +689,7 @@ katana -u https://tesla.com -f email,phone
*`-store-field`*
---

To compliment `field` option which is useful to filter output at run time, there is `-sf, -store-fields` option which works exactly like field option except instead of filtering, it stores all the information on the disk under `katana_field` directory sorted by target url.
To compliment `field` option which is useful to filter output at run time, there is `-sf, -store-fields` option which works exactly like field option except instead of filtering, it stores all the information on the disk under `katana_field` directory sorted by target url. Use `-sfd` or `-store-field-dir` to store data in a different location.

```
katana -u https://tesla.com -sf key,fqdn,qurl -silent
Expand Down
1 change: 1 addition & 0 deletions cmd/katana/main.go
Original file line number Diff line number Diff line change
Expand Up @@ -177,6 +177,7 @@ pipelines offering both headless and non-headless crawling.`)
flagSet.BoolVarP(&options.StoreResponse, "store-response", "sr", false, "store http requests/responses"),
flagSet.StringVarP(&options.StoreResponseDir, "store-response-dir", "srd", "", "store http requests/responses to custom directory"),
flagSet.BoolVarP(&options.NoClobber, "no-clobber", "ncb", false, "do not overwrite output file"),
flagSet.StringVarP(&options.StoreFieldDir, "store-field-dir", "sfd", "", "store per-host field to custom directory"),
flagSet.BoolVarP(&options.OmitRaw, "omit-raw", "or", false, "omit raw requests/responses from jsonl output"),
flagSet.BoolVarP(&options.OmitBody, "omit-body", "ob", false, "omit response body from jsonl output"),
flagSet.BoolVarP(&options.JSON, "jsonl", "j", false, "write output in jsonl format"),
Expand Down
3 changes: 1 addition & 2 deletions pkg/output/fields.go
Original file line number Diff line number Diff line change
Expand Up @@ -5,7 +5,6 @@ import (
"net/url"
"os"
"path"
"path/filepath"
"strings"

"github.com/projectdiscovery/gologger"
Expand Down Expand Up @@ -85,7 +84,7 @@ func storeFields(output *Result, storeFields []string) {
}

func appendToFileField(parsed *url.URL, field, data string) {
file, err := os.OpenFile(filepath.Join(storeFieldsDirectory, fmt.Sprintf("%s_%s_%s.txt", parsed.Scheme, parsed.Hostname(), field)), os.O_RDWR|os.O_CREATE|os.O_APPEND, 0666)
file, err := os.OpenFile(path.Join(storeFieldDir, fmt.Sprintf("%s_%s_%s.txt", parsed.Scheme, parsed.Hostname(), field)), os.O_RDWR|os.O_CREATE|os.O_APPEND, 0666)
if err != nil {
return
}
Expand Down
1 change: 1 addition & 0 deletions pkg/output/options.go
Original file line number Diff line number Diff line change
Expand Up @@ -19,6 +19,7 @@ type Options struct {
Fields string
StoreFields string
StoreResponseDir string
StoreFieldDir string
FieldConfig string
ErrorLogFile string
MatchRegex []*regexp.Regexp
Expand Down
12 changes: 8 additions & 4 deletions pkg/output/output.go
Original file line number Diff line number Diff line change
Expand Up @@ -21,12 +21,12 @@ import (
)

const (
storeFieldsDirectory = "katana_field"
indexFile = "index.txt"
DefaultResponseDir = "katana_response"
indexFile = "index.txt"
DefaultResponseDir = "katana_response"
)

var (
storeFieldDir = "katana_field"
decolorizerRegex = regexp.MustCompile(`\x1B\[[0-9;]*[a-zA-Z]`)
)

Expand Down Expand Up @@ -80,6 +80,10 @@ func New(options Options) (Writer, error) {
outputMatchCondition: options.OutputMatchCondition,
outputFilterCondition: options.OutputFilterCondition,
}

if options.StoreFieldDir != "" {
storeFieldDir = options.StoreFieldDir
}
// if fieldConfig empty get the default file
if options.FieldConfig == "" {
var err error
Expand All @@ -103,7 +107,7 @@ func New(options Options) (Writer, error) {
}
}
if options.StoreFields != "" {
_ = os.MkdirAll(storeFieldsDirectory, os.ModePerm)
_ = os.MkdirAll(storeFieldDir, os.ModePerm)
if err := validateFieldNames(options.StoreFields); err != nil {
return nil, errorutil.NewWithTag("output", "could not validate store fields").Wrap(err)
}
Expand Down
1 change: 1 addition & 0 deletions pkg/types/crawler_options.go
Original file line number Diff line number Diff line change
Expand Up @@ -70,6 +70,7 @@ func NewCrawlerOptions(options *Options) (*CrawlerOptions, error) {
StoreFields: options.StoreFields,
StoreResponseDir: options.StoreResponseDir,
NoClobber: options.NoClobber,
StoreFieldDir: options.StoreFieldDir,
OmitRaw: options.OmitRaw,
OmitBody: options.OmitBody,
FieldConfig: options.FieldConfig,
Expand Down
2 changes: 2 additions & 0 deletions pkg/types/options.go
Original file line number Diff line number Diff line change
Expand Up @@ -119,6 +119,8 @@ type Options struct {
StoreResponseDir string
// NoClobber specifies if katana should overwrite existing output files
NoClobber bool
// StoreFieldDir specifies if katana should use a custom directory to store fields
StoreFieldDir string
// OmitRaw omits raw requests/responses from the output
OmitRaw bool
// OmitBody omits the response body from the output
Expand Down

0 comments on commit 5ae80ae

Please sign in to comment.