Trim white space prefix in line of robots.txt

hakluke · Feb 20, 2021 · 9ba0782 · 9ba0782
1 parent e39a514
commit 9ba0782
Showing 1 changed file with 1 addition and 1 deletion.
diff --git a/pkg/collector/collector.go b/pkg/collector/collector.go
@@ -346,7 +346,7 @@ func (c *Collector) parseRobots(url string, reqsMade *syncList) {
 
         for _, line := range lines {
                 if re.MatchString(line) {
-                        urlstring := re.ReplaceAllString(line, "")
+                        urlstring := strings.TrimLeft(re.ReplaceAllString(line, ""), " ")
                         if c.conf.IncludeRobots || c.conf.IncludeAll {
                                 _ = c.recordIfInScope(c.au.BrightMagenta("[robots]"), url, url+urlstring, reqsMade)
                         }