updates

internal/engine/filters.go

@@ -133,7 +133,25 @@ func DedupFilter() func(in <-chan internal.Domain, c *config.Config) <-chan inte
 					out <- domain
 				}
 			}
-			c.Count(len(variants))
+			close(out)
+		}()
+		return out
+	}
+}
+
+func GetTotal() func(in <-chan internal.Domain, c *config.Config) <-chan internal.Domain {
+	return func(in <-chan internal.Domain, c *config.Config) <-chan internal.Domain {
+		out := make(chan internal.Domain)
+
+		go func() {
+			var count int
+			for domain := range in {
+				count++
+
+				out <- domain
+
+			}
+			c.Count(count)
 			close(out)
 		}()
 		return out

internal/engine/processor.go

@@ -382,7 +382,6 @@ func (u *Urlinsane) CollectorChain(funcs []internal.Collector, in <-chan interna
 				ctx, cancel = context.WithCancel(context.Background())
 			}
 
-			// acc := NewAccumulator(out)
 			u.runner(ctx, xfunc, variant, out)
 			cancel()
 		}
@@ -449,8 +448,10 @@ func (u *Urlinsane) Progress(in <-chan internal.Domain) <-chan internal.Domain {
 func (u *Urlinsane) Output(in <-chan internal.Domain) {
 	logger := log.WithFields(log.Fields{"output": u.cfg.Output().Id()})
 	output := u.cfg.Output()
+	var total int64
 
 	for c := range in {
+		total++
 		logger.WithFields(log.Fields{"d": c.String()}).
 			Debug("Sending to output")
 
@@ -486,13 +487,53 @@ func (u *Urlinsane) Output(in <-chan internal.Domain) {
 		summary := map[string]string{
 			"  TIME:":                             u.elapsed.String(),
 			text.FgGreen.Sprintf("%s", "  LIVE:"): fmt.Sprintf("%d", u.live),
-			text.FgRed.Sprintf("%s", "  OFFLINE"): fmt.Sprintf("%d", u.total-u.live),
-			"  TOTAL:":                            fmt.Sprintf("%d", u.total),
+			text.FgRed.Sprintf("%s", "  OFFLINE"): fmt.Sprintf("%d", total-u.live),
+			"  TOTAL:":                            fmt.Sprintf("%d", total),
 		}
 		output.Summary(summary)
 	}
 }
 
+func (u *Urlinsane) Details(in <-chan internal.Domain) {
+	// logger := log.WithFields(log.Fields{"output": u.cfg.Output().Id()})
+	// output := u.cfg.Output()
+
+	// for c := range in {
+	// 	logger.WithFields(log.Fields{"d": c.String()}).
+	// 		Debug("Sending to output")
+
+	// 	// Save domain to directory
+	// 	if u.cfg.AssetDir() != "" {
+	// 		if c.Live() {
+	// 			if err := c.Save(u.cfg.AssetDir()); err != nil {
+	// 				logger.Error("Saving: ", err)
+	// 			}
+	// 		}
+
+	// 	}
+	// }
+
+	// // Writes output if it can't stream
+	// output.Write()
+
+	// // Save typo records collected by the output plugin
+	// if fname := u.cfg.File(); fname != "" {
+	// 	output.Save(fname)
+	// }
+
+	// // Print summary
+	// if u.cfg.Summary() {
+	// 	u.elapsed = time.Since(u.started)
+	// 	summary := map[string]string{
+	// 		"  TIME:":                             u.elapsed.String(),
+	// 		text.FgGreen.Sprintf("%s", "  LIVE:"): fmt.Sprintf("%d", u.live),
+	// 		text.FgRed.Sprintf("%s", "  OFFLINE"): fmt.Sprintf("%d", total-u.live),
+	// 		"  TOTAL:":                            fmt.Sprintf("%d", total),
+	// 	}
+	// 	output.Summary(summary)
+	// }
+}
+
 func (u *Urlinsane) Close() {
 	// Initialize information plugins if needed
 	for _, info := range u.cfg.Collectors() {
@@ -528,6 +569,36 @@ func (u *Urlinsane) Execute() (err error) {
 	return
 }
 
+func (u *Urlinsane) Scan() (err error) {
+	typos := u.Init()
+	typos = u.Target(typos)
+	typos = u.Algorithms(typos)
+	typos = u.Constraints(typos,
+		DedupFilter,
+		RegexFilter,
+		LevenshteinFilter,
+		ReadCacheFilter,
+	)
+	typos = u.Collectors(typos)
+	typos = u.WriteCache(typos)
+	typos = u.PostFilters(typos)
+	typos = u.Analyzers(typos)
+	typos = u.Progress(typos)
+	u.Output(typos)
+	u.Close()
+
+	return
+}
+
+func (u *Urlinsane) Fetch() (err error) {
+	typos := u.Init()
+	typos = u.Target(typos)
+	typos = u.Collectors(typos)
+	u.Details(typos)
+	u.Close()
+	return
+}
+
 func (u *Urlinsane) Stream() <-chan internal.Domain {
 	typos := u.Init()
 	typos = u.Target(typos)
@@ -537,6 +608,7 @@ func (u *Urlinsane) Stream() <-chan internal.Domain {
 		RegexFilter,
 		LevenshteinFilter,
 		ReadCacheFilter,
+		GetTotal,
 	)
 	typos = u.Collectors(typos)
 	typos = u.WriteCache(typos)

internal/plugins/collectors/ns/ns.go

@@ -72,9 +72,11 @@ func (i *Plugin) Exec(acc internal.Accumulator) (err error) {
 	for _, record := range records {
 		dns.Add("NS", 0, record.Host)
 	}
-	acc.SetMeta("NS", dns.String("NS"))
-	acc.SetJson("DNS", dns.Json())
-	acc.Domain().Live(true)
+	if len(records) > 0 {
+		acc.SetMeta("NS", dns.String("NS"))
+		acc.SetJson("DNS", dns.Json())
+		acc.Domain().Live(true)
+	}
 
 	return acc.Next()
 }

internal/plugins/collectors/web/web.go

@@ -134,6 +134,8 @@ func (p *Plugin) Exec(acc internal.Accumulator) (err error) {
 
 		})
 
+		// p.client.Visit(fmt.Sprintf("http://%s/robot.txt", acc.Domain().String()))
+		// p.client.Visit(fmt.Sprintf("https://%s/robot.txt", acc.Domain().String()))
 		p.client.Visit(fmt.Sprintf("http://%s", acc.Domain().String()))
 		p.client.Visit(fmt.Sprintf("https://%s", acc.Domain().String()))