Micropilot

.gitignore

@@ -1,2 +1,3 @@
 *.swp
 *~
+packages/micropilot

README.md

@@ -1,8 +1,13 @@
 For users: https://github.com/mozilla/blushproof/wiki
 
 For developers:
+Requirements: nodejs, volo version 0.2.8 or higher
+
 To checkout:  
+    `npm install -g volo`
     `git clone https://github.com/mozilla/blushproof`  
+    `cd blushproof`
+    `volo add micropilot packages/micropilot`
     `cd .git`  
     `rm -rf hooks`  
     `ln -s ../hooks .`

lib/bpCategorizer.js

@@ -152,7 +152,6 @@ function whitelistCommon(aKey, aCategory, aWhitelistMap, aBlushlistMap) {
  */
 exports.isHostWhitelisted = function isHostWhitelisted(aHost) {
   let key = bpUtil.getKeyForHost(aHost);
-  console.log("Checking host", key);
   return (ss.storage.whitelistedDomains[key] ||
           isCategoryWhitelisted(getCategoryForHost(aHost)));
 }
@@ -163,7 +162,6 @@ exports.isHostWhitelisted = function isHostWhitelisted(aHost) {
  */
 exports.isQueryWhitelisted = function isQueryWhitelisted(aQuery) {
   let key = bpUtil.getKeyForQuery(aQuery);
-  console.log("Checking query", key);
   return (ss.storage.whitelistedQueries[key] ||
           isCategoryWhitelisted(getCategoryForQuery(aQuery)));
 }

lib/bpContentPolicy.js

@@ -11,6 +11,7 @@ let ss = require("simple-storage");
 let bpCategorizer = require("bpCategorizer");
 let bpUI = require("bpUI");
 let bpUtil = require("bpUtil");
+let { recordEvent, monitor, kEvents } = require("monitor");
 
 /**
  * Returns true if aWindow is in private browsing mode.
@@ -41,16 +42,25 @@ exports.bpContentPolicy = Class({
       return Ci.nsIContentPolicy.ACCEPT;
     }
 
-    // Ignore whitelisted queries and URLs
+    // Ignore whitelisted queries and URLs.
+    if (bpCategorizer.isHostWhitelisted(aContentLocation.host)) {
+      recordEvent(kEvents.WHITELISTED_SITE);
+      return Ci.nsIContentPolicy.ACCEPT;
+    }
+
     let query = bpUtil.getSearchTermFromURI(aContentLocation);
-    if (bpCategorizer.isHostWhitelisted(aContentLocation.host) ||
-        bpCategorizer.isQueryWhitelisted(query)) {
+    if (bpCategorizer.isQueryWhitelisted(query)) {
+      recordEvent(kEvents.WHITELISTED_QUERY);
       return Ci.nsIContentPolicy.ACCEPT;
     }
 
-    let category = bpCategorizer.getCategoryForHost(aContentLocation.host);
-    // Ignore non-blushy requests
-    if (!category && !bpCategorizer.getCategoryForQuery(query)) {
+    // Check against the blushlists.
+    if (bpCategorizer.getCategoryForHost(aContentLocation.host)) {
+      recordEvent(kEvents.BLUSHY_SITE);
+    } else if (bpCategorizer.getCategoryForQuery(query)) {
+      recordEvent(kEvents.BLUSHY_QUERY);
+    } else {
+      // Ignore non-blushy requests
       return Ci.nsIContentPolicy.ACCEPT;
     }
 
@@ -71,11 +81,9 @@ exports.bpContentPolicy = Class({
 
     // Reject and prompt if we're not in a private window.
     if (!isWindowPrivate(win)) {
-      console.log("stopping load for non-private window");
       bpUI.handleNavigation(win, aContentLocation);
       return Ci.nsIContentPolicy.REJECT_TYPE;
     }
-    console.log("in private window - not stopping load");
     return Ci.nsIContentPolicy.ACCEPT;
   },
 

lib/bpUI.js

@@ -14,14 +14,15 @@ const BROWSERURL = "chrome://browser/content/browser.xul"
 let bpContentPolicy = require("bpContentPolicy");
 let bpCategorizer = require("bpCategorizer");
 let bpUtil = require("bpUtil");
+let { recordEvent, monitor, kEvents } = require("monitor");
+
 const { Cc, Ci, Cu } = require("chrome");
 // The second param avoids polluting the scope
 const { ForgetAboutSite } = Cu.import("resource://gre/modules/ForgetAboutSite.jsm", {});
 
 let blushPanel = panel.Panel({
   contentURL: data.url("blushthis.html"),
   onMessage: function(aMessage) {
-    console.log("received message", aMessage);
     let host = utils.getMostRecentBrowserWindow().gBrowser.selectedBrowser
                  .contentWindow.location.host;
     // Unfortunately, [current browser].contentWindow.location is not an
@@ -30,9 +31,11 @@ let blushPanel = panel.Panel({
     let domain = normalizeHost(host);
     if (aMessage == "blush") {
       bpCategorizer.addToBlushlist(domain);
+      recordEvent(kEvents.ADD_BLUSHLIST);
       this.hide();
     } else if (aMessage == "forget") {
       ForgetAboutSite.removeDataFromDomain(domain);
+      recordEvent(kEvents.FORGET_SITE);
     } else {
       this.hide();
     }
@@ -100,10 +103,11 @@ function raiseConsent(aWindow, aURI) {
     contentURL: data.url('consent.html'),
     onMessage: function(aMessage) {
       if (aMessage == "openInPrivate") {
-        console.log("Opening", aURI.spec, "in private window");
+        recordEvent(kEvents.OPEN_PRIVATE);
         aWindow.openDialog(BROWSERURL, null, "chrome,all,dialog=no,private",
                            aURI.spec);
       } else if (aMessage == "continue") {
+        recordEvent(kEvents.OPEN_NORMAL);
         this.hide();
         let query = bpUtil.getSearchTermFromURI(aURI);
         if (bpCategorizer.getCategoryForQuery(query)) {

lib/bpUtil.js

@@ -80,13 +80,11 @@ function getSearchTermFromURI(aURI) {
   // Strip off the path so we can parse just the query params. querystring
   // isn't very sophisticated, but this works for now.
   let q = path.substr(searchMap[searchProvider].path.length);
-  console.log("query string", q);
   q = querystring.parse(q);
   if (!q) {
     console.log("Couldn't parse", path);
     return "";
   }
-  console.log("query:", JSON.stringify(q));
   return decodeURI(q[searchMap[searchProvider].query]).toLowerCase()
     .replace("+", " ");
 }

lib/monitor.js

@@ -0,0 +1,57 @@
+// This module encapsulates all the micropilot logic. Other modules should only
+// ever have to use monitor.record.
+
+const micropilot = require("micropilot");
+const { storage } = require("simple-storage");
+const myprefs = require("simple-prefs").prefs;
+
+// Our AWS box
+const UPLOAD_URL = "https://107.22.82.223";
+// Upload every day in milliseconds
+const PULSE_INTERVAL = 24 * 60 * 60 * 1000;
+
+let monitor = micropilot.Micropilot("blushproof").start();
+/**
+ * A helper function to record time-stamped events. Times are modulo the
+ * most recent hour.
+ * @param eventName {string} The name of the event.
+ * @return promise
+ */
+//micropilot.Micropilot.prototype.recordEvent = function recordEvent(eventName) {
+exports.recordEvent = function recordEvent(eventName) {
+  let d = new Date();
+  d.setMinutes(0);
+  d.setSeconds(0);
+  d.setMilliseconds(0);
+  return monitor.record({timestamp: d.getTime() / 1000, event: eventName});
+};
+
+function uploadAndClear() {
+  // We might lose events between when this.upload() returns and this.clear()
+  return monitor.upload(UPLOAD_URL).then(monitor.clear);
+};
+
+// Only start the Fuse to upload results if reporting is enabled
+if (myprefs.enable_reporting) {
+  var f = micropilot.Fuse({
+    start: Date.now(),
+    // Run forever
+    duration: false,
+    // Upload daily
+    pulseinterval: PULSE_INTERVAL,
+    pulsefn: uploadAndClear
+  }).start();
+}
+
+exports.monitor = monitor;
+const kEventNames = exports.kEvents = {
+  ADD_BLUSHLIST: "add-blushlist",
+  BLUSHY_QUERY: "blushy-query",
+  BLUSHY_SITE: "blushy-site",
+  FORGET_SITE: "forget-site",
+  OPEN_NORMAL: "open-normal",
+  OPEN_PRIVATE: "open-private",
+  REMOVE_BLUSHLIST: "remove-blushlist",
+  WHITELISTED_QUERY: "whitelisted-query",
+  WHITELISTED_SITE: "whitelisted-site"
+};

package.json

@@ -5,5 +5,27 @@
   "description": "stop opening naughty sites accidentally",
   "author": "David Keeler <[email protected]>, Monica Chew <[email protected]>, Gregg Lind <[email protected]>",
   "license": "MPL 2.0",
-  "version": "0.1"
+  "version": "0.1",
+  "dependencies": [
+    "micropilot"
+  ],
+  "preferences": [
+    {
+      "type": "bool",
+      "title": "Enable debug messages",
+      "name": "micropilotlog",
+      "value": true
+    },
+    {
+      "type": "bool",
+      "title": "Report metrics",
+      "name": "enable_reporting",
+      "value": false
+    }
+  ],
+  "volo": {
+    "dependencies": {
+      "packages/micropilot": "github:gregglind/micropilot/v0.8"
+    }
+  }
 }