diff --git a/internal/engine/filters.go b/internal/engine/filters.go index 8dee0a06..5c6c08bf 100644 --- a/internal/engine/filters.go +++ b/internal/engine/filters.go @@ -133,7 +133,25 @@ func DedupFilter() func(in <-chan internal.Domain, c *config.Config) <-chan inte out <- domain } } - c.Count(len(variants)) + close(out) + }() + return out + } +} + +func GetTotal() func(in <-chan internal.Domain, c *config.Config) <-chan internal.Domain { + return func(in <-chan internal.Domain, c *config.Config) <-chan internal.Domain { + out := make(chan internal.Domain) + + go func() { + var count int + for domain := range in { + count++ + + out <- domain + + } + c.Count(count) close(out) }() return out diff --git a/internal/engine/processor.go b/internal/engine/processor.go index 109a6b53..821a1843 100644 --- a/internal/engine/processor.go +++ b/internal/engine/processor.go @@ -382,7 +382,6 @@ func (u *Urlinsane) CollectorChain(funcs []internal.Collector, in <-chan interna ctx, cancel = context.WithCancel(context.Background()) } - // acc := NewAccumulator(out) u.runner(ctx, xfunc, variant, out) cancel() } @@ -449,8 +448,10 @@ func (u *Urlinsane) Progress(in <-chan internal.Domain) <-chan internal.Domain { func (u *Urlinsane) Output(in <-chan internal.Domain) { logger := log.WithFields(log.Fields{"output": u.cfg.Output().Id()}) output := u.cfg.Output() + var total int64 for c := range in { + total++ logger.WithFields(log.Fields{"d": c.String()}). Debug("Sending to output") @@ -486,13 +487,53 @@ func (u *Urlinsane) Output(in <-chan internal.Domain) { summary := map[string]string{ " TIME:": u.elapsed.String(), text.FgGreen.Sprintf("%s", " LIVE:"): fmt.Sprintf("%d", u.live), - text.FgRed.Sprintf("%s", " OFFLINE"): fmt.Sprintf("%d", u.total-u.live), - " TOTAL:": fmt.Sprintf("%d", u.total), + text.FgRed.Sprintf("%s", " OFFLINE"): fmt.Sprintf("%d", total-u.live), + " TOTAL:": fmt.Sprintf("%d", total), } output.Summary(summary) } } +func (u *Urlinsane) Details(in <-chan internal.Domain) { + // logger := log.WithFields(log.Fields{"output": u.cfg.Output().Id()}) + // output := u.cfg.Output() + + // for c := range in { + // logger.WithFields(log.Fields{"d": c.String()}). + // Debug("Sending to output") + + // // Save domain to directory + // if u.cfg.AssetDir() != "" { + // if c.Live() { + // if err := c.Save(u.cfg.AssetDir()); err != nil { + // logger.Error("Saving: ", err) + // } + // } + + // } + // } + + // // Writes output if it can't stream + // output.Write() + + // // Save typo records collected by the output plugin + // if fname := u.cfg.File(); fname != "" { + // output.Save(fname) + // } + + // // Print summary + // if u.cfg.Summary() { + // u.elapsed = time.Since(u.started) + // summary := map[string]string{ + // " TIME:": u.elapsed.String(), + // text.FgGreen.Sprintf("%s", " LIVE:"): fmt.Sprintf("%d", u.live), + // text.FgRed.Sprintf("%s", " OFFLINE"): fmt.Sprintf("%d", total-u.live), + // " TOTAL:": fmt.Sprintf("%d", total), + // } + // output.Summary(summary) + // } +} + func (u *Urlinsane) Close() { // Initialize information plugins if needed for _, info := range u.cfg.Collectors() { @@ -528,6 +569,36 @@ func (u *Urlinsane) Execute() (err error) { return } +func (u *Urlinsane) Scan() (err error) { + typos := u.Init() + typos = u.Target(typos) + typos = u.Algorithms(typos) + typos = u.Constraints(typos, + DedupFilter, + RegexFilter, + LevenshteinFilter, + ReadCacheFilter, + ) + typos = u.Collectors(typos) + typos = u.WriteCache(typos) + typos = u.PostFilters(typos) + typos = u.Analyzers(typos) + typos = u.Progress(typos) + u.Output(typos) + u.Close() + + return +} + +func (u *Urlinsane) Fetch() (err error) { + typos := u.Init() + typos = u.Target(typos) + typos = u.Collectors(typos) + u.Details(typos) + u.Close() + return +} + func (u *Urlinsane) Stream() <-chan internal.Domain { typos := u.Init() typos = u.Target(typos) @@ -537,6 +608,7 @@ func (u *Urlinsane) Stream() <-chan internal.Domain { RegexFilter, LevenshteinFilter, ReadCacheFilter, + GetTotal, ) typos = u.Collectors(typos) typos = u.WriteCache(typos) diff --git a/internal/plugins/collectors/ns/ns.go b/internal/plugins/collectors/ns/ns.go index b5c78062..929d4a94 100644 --- a/internal/plugins/collectors/ns/ns.go +++ b/internal/plugins/collectors/ns/ns.go @@ -72,9 +72,11 @@ func (i *Plugin) Exec(acc internal.Accumulator) (err error) { for _, record := range records { dns.Add("NS", 0, record.Host) } - acc.SetMeta("NS", dns.String("NS")) - acc.SetJson("DNS", dns.Json()) - acc.Domain().Live(true) + if len(records) > 0 { + acc.SetMeta("NS", dns.String("NS")) + acc.SetJson("DNS", dns.Json()) + acc.Domain().Live(true) + } return acc.Next() } diff --git a/internal/plugins/collectors/web/web.go b/internal/plugins/collectors/web/web.go index b2a7f3f5..c1a30c1f 100644 --- a/internal/plugins/collectors/web/web.go +++ b/internal/plugins/collectors/web/web.go @@ -134,6 +134,8 @@ func (p *Plugin) Exec(acc internal.Accumulator) (err error) { }) + // p.client.Visit(fmt.Sprintf("http://%s/robot.txt", acc.Domain().String())) + // p.client.Visit(fmt.Sprintf("https://%s/robot.txt", acc.Domain().String())) p.client.Visit(fmt.Sprintf("http://%s", acc.Domain().String())) p.client.Visit(fmt.Sprintf("https://%s", acc.Domain().String()))