"uniq" history entries before using them as a query source, as a history

may contain dozens of subsequent entries for the same hostname.

history/filter.go

@@ -4,6 +4,7 @@ package history
 import (
 	"code.google.com/p/go.net/publicsuffix"
 	"log"
+	"math/rand"
 	"net/url"
 	"regexp"
 )
@@ -26,8 +27,8 @@ func isPossiblyInternal(addr string) bool {
 	return false
 }
 
-// Filter out external hostnames from history, with a limit of X records (may be 0).
-func ExternalHostnames(entries []string, limit int) (hostnames []string) {
+// Filter out external hostnames from history
+func ExternalHostnames(entries []string) (hostnames []string) {
 	counter := 0
 
 	for _, uString := range entries {
@@ -38,11 +39,37 @@ func ExternalHostnames(entries []string, limit int) (hostnames []string) {
 		}
 		if !isPossiblyInternal(u.Host) {
 			counter += 1
-			if limit > 0 && counter > limit {
-				return
-			}
 			hostnames = append(hostnames, u.Host)
 		}
 	}
 	return
 }
+
+// Filter input array for unique entries.
+func Uniq(input []string) (output []string) {
+	last := ""
+	for _, i := range input {
+		if i != last {
+			output = append(output, i)
+		}
+	}
+	return
+}
+
+// Randomly select X number of entries.
+func Random(count int, input []string) (output []string) {
+	selected := make(map[int]bool)
+
+	for {
+		if len(selected) >= count {
+			return
+		}
+		index := rand.Intn(len(input))
+		// If we have already picked this number, re-roll.
+		if _, exists := selected[index]; exists == true {
+			continue
+		}
+		output = append(output, input[index])
+		selected[index] = true
+	}
+}

history/history.go

@@ -47,7 +47,7 @@ func Chrome(days int) (urls []string, err error) {
 		 LEFT JOIN urls ON visits.url = urls.id
 		 WHERE (visit_time - 11644473600000000 >
 			    strftime('%%s', date('now', '-%d day')) * 1000000)
-		 ORDER BY RANDOM();`, days)
+		 ORDER BY visit_time DESC`, days)
 
 	for _, p := range paths {
 		path := os.ExpandEnv(p)

ui/ui.go

@@ -30,7 +30,7 @@ func Index(w http.ResponseWriter, r *http.Request) {
 	}
 
 	q := dnsqueue.StartQueue(QUEUE_LENGTH, WORKERS)
-	hostnames := history.ExternalHostnames(records, 64)
+	hostnames := history.Random(16, history.Uniq(history.ExternalHostnames(records)))
 
 	for _, record := range hostnames {
 		q.Add("8.8.8.8:53", "A", record+".")