fix nasty soft 404 bug

librecursebuster/logic.go

@@ -1,6 +1,7 @@
 package librecursebuster
 
 import (
+	"bytes"
 	"fmt"
 	"io/ioutil"
 	"net/http"
@@ -171,7 +172,8 @@ func (gState *State) dirBust(page SpiderPage) {
 		h, _, res := gState.evaluateURL(gState.Methods[0], page.URL+RandString(), gState.Client)
 		//fmt.Println(page.URL, h, res)
 		if res { //true response indicates a good response for a guid path, unlikely good
-			if detectSoft404(h, gState.Hosts.Get404(u.Host), gState.Cfg.Ratio404) {
+			is404, _ := detectSoft404(h, gState.Hosts.Get404(u.Host), gState.Cfg.Ratio404)
+			if is404 {
 				//it's a soft404 probably, guess we can continue (this logic seems wrong??)
 			} else {
 				gState.PrintOutput(
@@ -277,6 +279,7 @@ func (gState *State) StartBusting(randURL string, u url.URL) {
 			Debug, 2,
 		)
 		content, _ := ioutil.ReadAll(resp.Body)
+		resp.Body = ioutil.NopCloser(bytes.NewBuffer(content))
 		gState.Hosts.AddSoft404Content(u.Host, content, resp) // Soft404ResponseBody = xx
 	} else {
 		<-gState.Chans.workersChan

librecursebuster/main_test.go

@@ -2,6 +2,7 @@ package librecursebuster
 
 import (
 	"fmt"
+	"net/http"
 	"net/url"
 	"os"
 	"strings"
@@ -59,44 +60,53 @@ func TestBasicFunctionality(t *testing.T) {
 	}
 	for _, i := range ok200 {
 		tested = append(tested, i)
-		if x, ok := found[i]; !ok || !x {
-			panic("Did not find " + i)
+		if x, ok := found[i]; !ok || x == nil {
+			t.Error("Did not find " + i)
 		}
 	}
 	ok300 := []string{
 		"/b", "/b/c",
 	}
 	for _, i := range ok300 {
 		tested = append(tested, i)
-		if x, ok := found[i]; !ok || !x {
-			panic("Did not find " + i)
+		if x, ok := found[i]; !ok || x == nil {
+			t.Error("Did not find " + i)
 		}
 	}
 	ok400 := []string{
 		"/a/b/c/", "/a/b/c/d",
 	}
 	for _, i := range ok400 {
 		tested = append(tested, i)
-		if x, ok := found[i]; !ok || !x {
-			panic("Did not find " + i)
+		if x, ok := found[i]; !ok || x == nil {
+			t.Error("Did not find " + i)
 		}
 	}
 	ok500 := []string{
 		"/c/d", "/c", "/c/",
 	}
 	for _, i := range ok500 {
 		tested = append(tested, i)
-		if x, ok := found[i]; !ok || !x {
-			panic("Did not find " + i)
+		if x, ok := found[i]; !ok || x == nil {
+			t.Error("Did not find " + i)
 		}
 	}
 
 	//check for values that should not have been found
 	for k := range found {
 		if strings.Contains(k, "z") {
-			panic("Found (but should not have) " + k)
+			t.Error("Found (but should not have) " + k)
 		}
 	}
+
+	if x, ok := found["/a/x"]; ok && x != nil {
+		t.Error("Found (but should not have) /a/x")
+	}
+
+	if x, ok := found["/a/y"]; ok && x != nil {
+		t.Error("Found (but should not have) /a/x")
+	}
+
 	close(finished)
 }
 
@@ -111,8 +121,8 @@ func TestAppendSlash(t *testing.T) {
 	found := postSetupTest(urlSlice, gState)
 	gState.Wait()
 
-	if x, ok := found["/appendslash/"]; !ok || !x {
-		panic("didn't find it?")
+	if x, ok := found["/appendslash/"]; !ok || x == nil {
+		t.Error("didn't find it?")
 	}
 	close(finished)
 }
@@ -128,8 +138,8 @@ func TestBasicAuth(t *testing.T) {
 	found := postSetupTest(urlSlice, gState)
 	gState.Wait()
 
-	if x, ok := found["/a/b/c/basicauth"]; !ok || !x {
-		panic("Failed basic auth test!")
+	if x, ok := found["/a/b/c/basicauth"]; !ok || x == nil {
+		t.Error("Failed basic auth test!")
 	}
 
 }
@@ -146,7 +156,7 @@ func TestBadCodes(t *testing.T) {
 
 	for x := range found {
 		if strings.Contains(x, "badcode") {
-			panic("Failed bad header code test")
+			t.Error("Failed bad header code test")
 		}
 	}
 
@@ -165,7 +175,7 @@ func TestBadHeaders(t *testing.T) {
 
 	for x := range found {
 		if strings.Contains(x, "badheader") {
-			panic("Failed bad header code test")
+			t.Error("Failed bad header code test")
 		}
 	}
 
@@ -184,16 +194,16 @@ func TestAjax(t *testing.T) {
 	found := postSetupTest(urlSlice, gState)
 	gState.Wait()
 
-	if x, ok := found["/ajaxonly"]; !ok || !x {
-		panic("Failed ajax header check")
+	if x, ok := found["/ajaxonly"]; !ok || x == nil {
+		t.Error("Failed ajax header check 1")
 	}
 
-	if x, ok := found["/ajaxpost"]; !ok || !x {
-		panic("Failed ajax header check")
+	if x, ok := found["/ajaxpost"]; !ok || x == nil {
+		t.Error("Failed ajax header check 2")
 	}
 
-	if x, ok := found["/onlynoajax"]; ok || x {
-		panic("Failed ajax header check")
+	if x, ok := found["/onlynoajax"]; ok && x != nil {
+		t.Error("Failed ajax header check 3")
 	}
 
 }
@@ -211,8 +221,8 @@ func TestBodyContent(t *testing.T) {
 	found := postSetupTest(urlSlice, gState)
 	gState.Wait()
 
-	if x, ok := found["/postbody"]; !ok || !x {
-		panic("Failed body based request")
+	if x, ok := found["/postbody"]; !ok || x == nil {
+		t.Error("Failed body based request")
 	}
 }
 
@@ -227,12 +237,12 @@ func TestBlacklist(t *testing.T) {
 	found := postSetupTest(urlSlice, gState)
 	gState.Wait()
 
-	if x, ok := found["/a/b"]; ok || x {
-		panic("Failed blacklist testing1")
+	if x, ok := found["/a/b"]; ok && x != nil {
+		t.Error("Failed blacklist testing1")
 	}
 
-	if x, ok := found["/a/b/c"]; ok || x {
-		panic("Failed blacklist testing2")
+	if x, ok := found["/a/b/c"]; ok && x != nil {
+		t.Error("Failed blacklist testing2")
 	}
 }
 
@@ -246,8 +256,8 @@ func TestCookies(t *testing.T) {
 	found := postSetupTest(urlSlice, gState)
 	gState.Wait()
 
-	if x, ok := found["/cookiesonly"]; !ok || !x {
-		panic("Failed Cookie test")
+	if x, ok := found["/cookiesonly"]; !ok || x == nil {
+		t.Error("Failed Cookie test")
 	}
 }
 func TestExt(t *testing.T) {
@@ -259,20 +269,48 @@ func TestExt(t *testing.T) {
 	found := postSetupTest(urlSlice, gState)
 	gState.Wait()
 
-	if x, ok := found["/a.exe"]; !ok || !x {
-		panic("Failed Ext test1")
+	if x, ok := found["/a.exe"]; !ok || x == nil {
+		t.Error("Failed Ext test1")
+	}
+
+	if x, ok := found["/a.aspx"]; !ok || x == nil {
+		t.Error("Failed Ext test2")
+	}
+
+	if x, ok := found["/a.csv"]; !ok || x == nil {
+		t.Error("Failed Ext test3")
 	}
+}
+
+func TestOutAll(t *testing.T) {
+	// sets all responses as 'found' for the purposes of output. Should not pass any logic tests for adding additional URLs etc
+	t.Parallel()
+	finished := make(chan struct{})
+	cfg := getDefaultConfig()
+	cfg.ShowAll = true
+	gState, urlsSlice := preSetupTest(cfg, "2011", finished, t)
+	found := postSetupTest(urlsSlice, gState)
+	gState.Wait()
 
-	if x, ok := found["/a.aspx"]; !ok || !x {
-		panic("Failed Ext test2")
+	//Check the 404's were received and set as found
+	if x, ok := found["/a/x/c"]; ok && x != nil {
+		fmt.Println("/a/x/c:", x)
+		fmt.Println("/a/x", found["/a/x"])
+		t.Error("Failed OutAll Test 1, performed check on non-existent prefix")
 	}
 
-	if x, ok := found["/a.csv"]; !ok || !x {
-		panic("Failed Ext test3")
+	if x, ok := found["/x"]; ok && x != nil {
+		//have 404 response in found set
+		if x.StatusCode != 404 {
+			t.Error("Failed OutAll Test 3, got unexpected response recorded for 404 check")
+		}
+	} else {
+		//didn't have '/a/x' (soft 404) in found set
+		t.Error("Failed OutAll Test 2, did not have 404 response in found set")
 	}
 }
 
-func postSetupTest(urlSlice []string, gState *State) (found map[string]bool) {
+func postSetupTest(urlSlice []string, gState *State) (found map[string]*http.Response) {
 	//start up the management goroutines
 	go gState.ManageRequests()
 	go gState.ManageNewURLs()
@@ -306,7 +344,7 @@ func postSetupTest(urlSlice []string, gState *State) (found map[string]bool) {
 	}()
 
 	//use the found map to determine later on if we have found the expected URL's
-	found = make(map[string]bool)
+	found = make(map[string]*http.Response)
 	go func() {
 		t := time.NewTicker(1 * time.Second).C
 		for {
@@ -318,7 +356,7 @@ func postSetupTest(urlSlice []string, gState *State) (found map[string]bool) {
 					gState.wg.Done()
 					panic(e)
 				}
-				found[u.Path] = true
+				found[u.Path] = x.Result
 				gState.wg.Done()
 				//fmt.Println("CONFIRMED!", x)
 			case <-t:

librecursebuster/net.go

@@ -101,6 +101,7 @@ func (gState *State) HTTPReq(method, path string, client *http.Client) (resp *ht
 	}
 	defer resp.Body.Close()
 
+	//Set body to be readable again
 	body, err := ioutil.ReadAll(resp.Body)
 	if err != nil {
 		return resp, err
@@ -111,9 +112,7 @@ func (gState *State) HTTPReq(method, path string, client *http.Client) (resp *ht
 }
 
 func (gState *State) evaluateURL(method string, urlString string, client *http.Client) (headResp *http.Response, content []byte, success bool) {
-	success = true
-	//wg.Add(1)
-	//PrintOutput("EVALUATING:"+method+":"+urlString, Debug, 4, wg, printChan)
+
 	//optimize GET requests by sending a head first (it's cheaper)
 	if method == "GET" && !gState.Cfg.NoHead {
 		headResp, err := gState.HTTPReq("HEAD", urlString, client) //send a HEAD. Ignore body response
@@ -143,6 +142,7 @@ func (gState *State) evaluateURL(method string, urlString string, client *http.C
 
 	headResp, err := gState.HTTPReq(method, urlString, client)
 	content, _ = ioutil.ReadAll(headResp.Body)
+	headResp.Body = ioutil.NopCloser(bytes.NewBuffer(content))
 	<-gState.Chans.workersChan //done with the net thread
 	if err != nil {
 		success = false
@@ -151,12 +151,13 @@ func (gState *State) evaluateURL(method string, urlString string, client *http.C
 		return headResp, content, success
 	}
 
-	//Check if we care about it (header only) section
+	//Check if we care about it (response code only) section
 	if gState.BadResponses[headResp.StatusCode] {
 		success = false
 		return headResp, content, success
 	}
 
+	//check for bad headers in the response
 	if len(gState.Cfg.BadHeader) > 0 {
 		for _, x := range gState.Cfg.BadHeader {
 			spl := strings.Split(x, ":")
@@ -171,24 +172,18 @@ func (gState *State) evaluateURL(method string, urlString string, client *http.C
 			}
 		}
 	}
-	//if gState.BadHeaders[headResp.Header.]
-
-	//get content from validated path/file thing
-	if gState.Cfg.BurpMode {
-		gState.HTTPReq(method, urlString, gState.BurpClient)
-	}
 
 	//check we care about it (body only) section
 	//double check that it's not 404/error using smart blockchain AI tech
-	gState.PrintOutput(
-		fmt.Sprintf("Checking body for 404:\nContent: %v,\nSoft404:%v,\nResponse:%v",
-			string(content), string(gState.Hosts.Get404Body(headResp.Request.Host)),
-			detectSoft404(headResp, gState.Hosts.Get404(headResp.Request.Host), gState.Cfg.Ratio404)),
-		Debug, 4)
-	if detectSoft404(headResp, gState.Hosts.Get404(headResp.Request.Host), gState.Cfg.Ratio404) {
+	is404, _ := detectSoft404(headResp, gState.Hosts.Get404(headResp.Request.Host), gState.Cfg.Ratio404)
+	if is404 {
 		//seems to be a soft 404 lol
 		return headResp, content, false
 	}
+
+	if gState.Cfg.BurpMode {
+		gState.HTTPReq(method, urlString, gState.BurpClient)
+	}
 	return headResp, content, true
 }