From 5ae80aec1c2650054c0b3f27f671e3d40a5bbbdc Mon Sep 17 00:00:00 2001 From: Dogan Can Bakir <65292895+dogancanbakir@users.noreply.github.com> Date: Thu, 27 Jun 2024 12:16:40 +0300 Subject: [PATCH] add `-store-field-dir` flag (#877) --- README.md | 3 ++- cmd/katana/main.go | 1 + pkg/output/fields.go | 3 +-- pkg/output/options.go | 1 + pkg/output/output.go | 12 ++++++++---- pkg/types/crawler_options.go | 1 + pkg/types/options.go | 2 ++ 7 files changed, 16 insertions(+), 7 deletions(-) diff --git a/README.md b/README.md index 2011262e..e24af9b2 100644 --- a/README.md +++ b/README.md @@ -193,6 +193,7 @@ OUTPUT: -o, -output string file to write output to -sr, -store-response store http requests/responses -srd, -store-response-dir string store http requests/responses to custom directory + -sfd, -store-field-dir string store per-host field to custom directory -or, -omit-raw omit raw requests/responses from jsonl output -ob, -omit-body omit response body from jsonl output -j, -jsonl write output in jsonl format @@ -688,7 +689,7 @@ katana -u https://tesla.com -f email,phone *`-store-field`* --- -To compliment `field` option which is useful to filter output at run time, there is `-sf, -store-fields` option which works exactly like field option except instead of filtering, it stores all the information on the disk under `katana_field` directory sorted by target url. +To compliment `field` option which is useful to filter output at run time, there is `-sf, -store-fields` option which works exactly like field option except instead of filtering, it stores all the information on the disk under `katana_field` directory sorted by target url. Use `-sfd` or `-store-field-dir` to store data in a different location. ``` katana -u https://tesla.com -sf key,fqdn,qurl -silent diff --git a/cmd/katana/main.go b/cmd/katana/main.go index d6ce829f..338b4068 100644 --- a/cmd/katana/main.go +++ b/cmd/katana/main.go @@ -177,6 +177,7 @@ pipelines offering both headless and non-headless crawling.`) flagSet.BoolVarP(&options.StoreResponse, "store-response", "sr", false, "store http requests/responses"), flagSet.StringVarP(&options.StoreResponseDir, "store-response-dir", "srd", "", "store http requests/responses to custom directory"), flagSet.BoolVarP(&options.NoClobber, "no-clobber", "ncb", false, "do not overwrite output file"), + flagSet.StringVarP(&options.StoreFieldDir, "store-field-dir", "sfd", "", "store per-host field to custom directory"), flagSet.BoolVarP(&options.OmitRaw, "omit-raw", "or", false, "omit raw requests/responses from jsonl output"), flagSet.BoolVarP(&options.OmitBody, "omit-body", "ob", false, "omit response body from jsonl output"), flagSet.BoolVarP(&options.JSON, "jsonl", "j", false, "write output in jsonl format"), diff --git a/pkg/output/fields.go b/pkg/output/fields.go index ab8666b7..cf074a83 100644 --- a/pkg/output/fields.go +++ b/pkg/output/fields.go @@ -5,7 +5,6 @@ import ( "net/url" "os" "path" - "path/filepath" "strings" "github.com/projectdiscovery/gologger" @@ -85,7 +84,7 @@ func storeFields(output *Result, storeFields []string) { } func appendToFileField(parsed *url.URL, field, data string) { - file, err := os.OpenFile(filepath.Join(storeFieldsDirectory, fmt.Sprintf("%s_%s_%s.txt", parsed.Scheme, parsed.Hostname(), field)), os.O_RDWR|os.O_CREATE|os.O_APPEND, 0666) + file, err := os.OpenFile(path.Join(storeFieldDir, fmt.Sprintf("%s_%s_%s.txt", parsed.Scheme, parsed.Hostname(), field)), os.O_RDWR|os.O_CREATE|os.O_APPEND, 0666) if err != nil { return } diff --git a/pkg/output/options.go b/pkg/output/options.go index d519c353..99faa32a 100644 --- a/pkg/output/options.go +++ b/pkg/output/options.go @@ -19,6 +19,7 @@ type Options struct { Fields string StoreFields string StoreResponseDir string + StoreFieldDir string FieldConfig string ErrorLogFile string MatchRegex []*regexp.Regexp diff --git a/pkg/output/output.go b/pkg/output/output.go index 90cb65ee..0e0e1983 100644 --- a/pkg/output/output.go +++ b/pkg/output/output.go @@ -21,12 +21,12 @@ import ( ) const ( - storeFieldsDirectory = "katana_field" - indexFile = "index.txt" - DefaultResponseDir = "katana_response" + indexFile = "index.txt" + DefaultResponseDir = "katana_response" ) var ( + storeFieldDir = "katana_field" decolorizerRegex = regexp.MustCompile(`\x1B\[[0-9;]*[a-zA-Z]`) ) @@ -80,6 +80,10 @@ func New(options Options) (Writer, error) { outputMatchCondition: options.OutputMatchCondition, outputFilterCondition: options.OutputFilterCondition, } + + if options.StoreFieldDir != "" { + storeFieldDir = options.StoreFieldDir + } // if fieldConfig empty get the default file if options.FieldConfig == "" { var err error @@ -103,7 +107,7 @@ func New(options Options) (Writer, error) { } } if options.StoreFields != "" { - _ = os.MkdirAll(storeFieldsDirectory, os.ModePerm) + _ = os.MkdirAll(storeFieldDir, os.ModePerm) if err := validateFieldNames(options.StoreFields); err != nil { return nil, errorutil.NewWithTag("output", "could not validate store fields").Wrap(err) } diff --git a/pkg/types/crawler_options.go b/pkg/types/crawler_options.go index 05dc4c9d..86cb6592 100644 --- a/pkg/types/crawler_options.go +++ b/pkg/types/crawler_options.go @@ -70,6 +70,7 @@ func NewCrawlerOptions(options *Options) (*CrawlerOptions, error) { StoreFields: options.StoreFields, StoreResponseDir: options.StoreResponseDir, NoClobber: options.NoClobber, + StoreFieldDir: options.StoreFieldDir, OmitRaw: options.OmitRaw, OmitBody: options.OmitBody, FieldConfig: options.FieldConfig, diff --git a/pkg/types/options.go b/pkg/types/options.go index b402d628..6b8eefb1 100644 --- a/pkg/types/options.go +++ b/pkg/types/options.go @@ -119,6 +119,8 @@ type Options struct { StoreResponseDir string // NoClobber specifies if katana should overwrite existing output files NoClobber bool + // StoreFieldDir specifies if katana should use a custom directory to store fields + StoreFieldDir string // OmitRaw omits raw requests/responses from the output OmitRaw bool // OmitBody omits the response body from the output