Merge pull request #521 from privacy-tech-lab/issue-512

Fix watchlist problems

README.md

@@ -182,6 +182,10 @@ Privacy Pioneer makes a distinction between Fine Location and Coarse Location wi
 
 ipinfo.io is sent the user's IP address and returns information about their location based on that IP address. We take the user's Zip Code, Street Address, City, and Region from this and store it as an entry in the user's Watchlist to be looked for in new HTTP requests.
 
+If a user looks up a specific keyword from their watchlist (for example in a search bar or search engine), we will show notifications of this keyword being taken by a first party. For normal lookups (like keywords a user could be interested in), this will appear as strange, but it is important to know when a website is or is not encrypting user data when sending it to their backend APIs, since all web traffic has the potential to be monitored in transit across the internet.
+
+Privacy Pioneer will notify a user when any of their watchlist information has been seen in their network traffic if the user has enabled notifications on the browser and in the watchlist page of the extension. These notifications will occur 15 seconds after a site has loaded to let the page load most of its data. User's Custom Keywords (shown as "Keyword" in the extension) will generate a notification each time they appear in the user's web traffic, while all other keywords being seen will generate a notification once per session per website that collected or shared a user's information. Note that if multiple notifications would have qualified for all categories other than personal (eg. zip, region, and IP would have been notified), only one notification will be shown, with a "..." following the first to signify that this is a joint notification and the user should check the extension to see Privacy Pioneer's findings.
+
 ## 8. Extension Architecture
 
 An overview of the architecture of Privacy Pioneer is [available separately](https://github.com/privacy-tech-lab/privacy-pioneer/blob/main/architecture_overview.md). (The document is up to date as of its most recent commit date. Later architectural changes are not reflected.)
@@ -205,6 +209,9 @@ We thank the developers.
 
 - Some warnings may occur when you run `npm install --production=false`, but they will not negatively affect the compilation or execution of Privacy Pioneer.
 - When the overview page of Privacy Pioneer is open, data from websites visited after opening it will not be shown until the overview is refreshed.
+- Phone numbers do not currently generate notifications. This will be addressed soon.
+- IP addresses sometimes do not generate notifications. This will be addressed soon.
+- We do not look at the main HTML / main_frame type of data request in this extension. Thus, data that was loaded into the main body of the webpage, and not through an external query, will not generate evidence in Privacy Pioneer.
 
 ## 11. Thank You!
 

src/background/analysis/buildUserData/importSearchData.js

@@ -207,14 +207,21 @@ async function importData() {
     var normalEmails = [];
     var encodedEmails = {};
     user_store_dict[typeEnum.emailAddress].forEach(async (email) => {
+      const emailSet = setEmail(email);
+
       // add the normal email as a regex where the non alphanumeric characters are possibly changed or not present
-      normalEmails.push(new RegExp(buildGeneralRegex(email)));
+      const newEmail = createKeywordObj(
+        new RegExp(buildGeneralRegex(email)),
+        typeEnum.emailAddress,
+        watchlistHashGen(typeEnum.emailAddress, emailSet)
+      );
+      normalEmails.push(newEmail);
 
       // add encoded emails
-      const digestHex = await digestMessage(setEmail(email));
+      const digestHex = await digestMessage(emailSet);
       const base64Encoded = hexToBase64(digestHex);
       const urlBase64Encoded = encodeURIComponent(base64Encoded);
-      const origHash = watchlistHashGen(typeEnum.emailAddress, email);
+      const origHash = watchlistHashGen(typeEnum.emailAddress, emailSet);
       const base64EncodedObj = createKeywordObj(
         base64Encoded,
         typeEnum.emailAddress,

src/background/analysis/buildUserData/structureUserData.js

@@ -101,6 +101,7 @@ function createKeywordObj(keyword, typ, hash = null) {
 function hashUserDictValues(networkKeywords) {
   const excludedSet = new Set([
     typeEnum.phoneNumber,
+    typeEnum.emailAddress,
     typeEnum.encodedEmail,
     typeEnum.ipAddress,
     typeEnum.userKeyword,

src/background/analysis/interactDB/addEvidence.js

@@ -309,12 +309,34 @@ function updateFetchedDict(evidenceDict, e) {
     if (perm in evidence) {
       // if type is in the permission
       if (t in evidence[perm]) {
+        // user keywords only
+        if (t === "userKeyword") {
+          if (evidence[perm][t][reqUrl] === undefined) {
+            evidence[perm][t][reqUrl] = Array(e);
+          } else {
+            // if this exact keyword has been found on this website before, gives index, otherwise -1
+            let haveKeywordEv = evidence[perm][t][reqUrl].findIndex(
+              (el) => el["watchlistHash"] == e["watchlistHash"]
+            );
+            // if this exact keyword has been found on this website before, update timestamp
+            if (haveKeywordEv !== -1) {
+              evidence[perm][t][reqUrl][haveKeywordEv]["timestamp"] =
+                e["timestamp"];
+            } else {
+              // this exact keyword has not been found on this website before, updates evidence
+              evidence[perm][t][reqUrl].push(e);
+            }
+            return evidence;
+          }
+        }
+
+        // all other types
         let hardNo = reqUrl in evidence[perm][t]; //we have exactly this evidence already
         // if we have the evidence update its timestamp
         if (hardNo) {
           evidence[perm][t][reqUrl]["timestamp"] = e["timestamp"];
         }
-        // if we have less than 5 different reqUrl's for this permission and this is a unique reqUrl, we save the evidence
+        // if this is a unique reqUrl, we save the evidence
         if (!hardNo) {
           evidence[perm][t][reqUrl] = e;
         } else {
@@ -323,7 +345,9 @@ function updateFetchedDict(evidenceDict, e) {
       } else {
         // we don't have this type yet, so we initialize it
         evidence[perm][t] = {};
-        evidence[perm][t][reqUrl] = e;
+        t === "userKeyword"
+          ? (evidence[perm][t][reqUrl] = Array(e))
+          : (evidence[perm][t][reqUrl] = e);
       }
     } else {
       // we don't have this permission yet so we initialize
@@ -332,14 +356,18 @@ function updateFetchedDict(evidenceDict, e) {
       // init dict for permission type pair
       evidence[perm][t] = {};
 
-      evidence[perm][t][reqUrl] = e;
+      t === "userKeyword"
+        ? (evidence[perm][t][reqUrl] = Array(e))
+        : (evidence[perm][t][reqUrl] = e);
     }
   }
   // we have don't have this rootUrl yet. So we init evidence at this url
   else {
     evidence[perm] = {};
     evidence[perm][t] = {};
-    evidence[perm][t][reqUrl] = e;
+    t === "userKeyword"
+      ? (evidence[perm][t][reqUrl] = Array(e))
+      : (evidence[perm][t][reqUrl] = e);
   }
   return evidence;
 }

src/background/analysis/requestAnalysis/scanHTTP.js

@@ -81,6 +81,7 @@ function getAllEvidenceForRequest(request, userData) {
       arr.push(evList); // push the evidence to the arr
     }
   }
+
 
   executeAndPush(urlSearch(rootUrl, reqUrl, request.urlClassification));
 

src/background/background.js

@@ -18,7 +18,7 @@ import {
   setDefaultSettings,
 } from "../libs/indexed-db/settings/index.js";
 import { importData } from "./analysis/buildUserData/importSearchData.js";
-import runNotifications from "../libs/indexed-db/notifications";
+import { runNotifications } from "../libs/indexed-db/notifications";
 import Queue from "queue";
 import { getHostname } from "./analysis/utility/util.js";
 import { EVIDENCE_THRESHOLD, FIVE_SEC_IN_MILLIS } from "./analysis/constants";

src/libs/components/label-detail/components/item/index.js

@@ -11,19 +11,39 @@ import {
   privacyLabels,
   settingsModelsEnum,
 } from "../../../../../background/analysis/classModels";
-import {
-  getAnalyticsStatus,
-  settingsEnum,
-} from "../../../../indexed-db/settings";
+import { getAnalyticsStatus } from "../../../../indexed-db/settings";
 import { handleClick } from "../../../../indexed-db/getAnalytics";
 
+/**
+ * The function `formatItemUserKeywords` takes a request object and formats the userKeyword property by
+ * converting it into separate properties with unique keys.
+ * @param request - The `request` parameter is an object that contains different types of keywords. One
+ * of the types is "userKeyword", which is an array of user-defined keywords. The function
+ * `formatItemUserKeywords` takes this `request` object and formats it by appending an index number to
+ * each user keyword.
+ * @returns The function `formatItemUserKeywords` returns either the formatted `newReq` object if it is
+ * not empty, or the original `request` object if `newReq` is empty.
+ */
+const formatItemUserKeywords = (request) => {
+  var newReq = {};
+  for (const [type, eList] of Object.entries(request)) {
+    if (type === "userKeyword") {
+      for (var i = 0; i < eList.length; i++) {
+        newReq[type + i.toString()] = eList[i];
+      }
+    }
+  }
+  return Object.keys(newReq).length > 0 ? newReq : request;
+};
+
 /**
  * Display of badges with sub types and collapse containing description and evidence
  * @param {object} request our evidence object for this request
  * @param {string} url the request url
  * @param {string} label the associated label of this evidence
  */
 const Item = ({ request, url, label }) => {
+  request = formatItemUserKeywords(request);
   const [evidence, setEvidence] = useState({
     request: null,
     label: null,
@@ -68,35 +88,39 @@ const Item = ({ request, url, label }) => {
   return (
     <>
       <SBadgeGroup ref={containerRef}>
-        {Object.entries(request).map(([type, request]) => (
-          <SBadge
-            key={type}
-            className="badge"
-            onClick={(event) => {
-              inflateCollapse(event, request, type);
-              const getAnalysis = async () => {
-                const status = await getAnalyticsStatus();
-                if (status == true) {
-                  handleClick(
-                    /* Gets Description Button (Tracking Pixel, Fine Location, IP, etc) and Third Party */
-                    "Description Button: " +
-                      type.toString() +
-                      " Third Party: " +
+        {Object.entries(request).map(([type, request]) => {
+          const ktype = type;
+          type = type.startsWith("userKeyword") ? "userKeyword" : type;
+          return (
+            <SBadge
+              key={ktype}
+              className="badge"
+              onClick={(event) => {
+                inflateCollapse(event, request, type);
+                const getAnalysis = async () => {
+                  const status = await getAnalyticsStatus();
+                  if (status == true) {
+                    handleClick(
+                      /* Gets Description Button (Tracking Pixel, Fine Location, IP, etc) and Third Party */
+                      "Description Button: " +
+                        type.toString() +
+                        " Third Party: " +
+                        url.toString(),
+                      "ANY",
+                      settingsModelsEnum.notApplicable,
                       url.toString(),
-                    "ANY",
-                    settingsModelsEnum.notApplicable,
-                    url.toString(),
-                    settingsModelsEnum.notApplicable
-                  );
-                }
-              };
-              getAnalysis();
-            }}
-          >
-            {privacyLabels[label]["types"][type]["displayName"]}
-            {request.cookie ? ` 🍪` : null}
-          </SBadge>
-        ))}
+                      settingsModelsEnum.notApplicable
+                    );
+                  }
+                };
+                getAnalysis();
+              }}
+            >
+              {privacyLabels[label]["types"][type]["displayName"]}
+              {request.cookie ? ` 🍪` : null}
+            </SBadge>
+          );
+        })}
       </SBadgeGroup>
       <Evidence
         request={evidence.request}

src/libs/indexed-db/getIdbData.js

@@ -9,7 +9,6 @@ import {
   privacyLabels,
 } from "../../background/analysis/classModels";
 import { getExcludedLabels } from "../indexed-db/settings";
-import { IPINFO_ADDRESSKEY, IPINFO_IPKEY } from "../../background/analysis/buildUserData/importSearchData.js";
 
 /**
  * Get identified labels of website from indexedDB
@@ -47,7 +46,6 @@ export const getWebsiteLabels = async (website, excludedLabels = []) => {
   }
 };
 
-
 /**
  * Get identified labels of website from indexedDB from latest browsing session
  * Restucture to display in UI
@@ -58,14 +56,31 @@ export const getWebsiteLastVisitedEvidence = async (website) => {
     const labels = await getWebsiteLabels(website);
     const lastSeen = (await evidenceIDB.get(website)).lastSeen;
     var result = {};
-
-    for (const [label, value] of Object.entries(labels)) { 
-        for (const [url, typeVal] of Object.entries(value)) {
-          for (const [type, e] of Object.entries(typeVal)) {
+
+    for (const [label, value] of Object.entries(labels)) {
+      for (const [url, typeVal] of Object.entries(value)) {
+        for (const [type, e] of Object.entries(typeVal)) {
+          if (type === "userKeyword") {
+            for (const el of e) {
+              //Check if the evidence has been added recently
+              var isCurrentSessionEvidence = el["timestamp"] >= lastSeen - 60000;
+
+              if (isCurrentSessionEvidence) {
+                // Add label in data to object
+                if (!(label in result)) {
+                  result[label] = { [url]: { [type]: Array(el) } };
+                } else if (!(url in result[label])) {
+                  result[label][url] = { [type]: Array(el) };
+                } else {
+                  result[label][url][type].push(el);
+                }
+              }
+            }
+          } else {
             //Check if the evidence has been added recently
             var isCurrentSessionEvidence = e["timestamp"] >= lastSeen - 60000;
-            
-            if ( isCurrentSessionEvidence) {
+
+            if (isCurrentSessionEvidence) {
               // Add label in data to object
               if (!(label in result)) {
                 result[label] = { [url]: { [type]: e } };
@@ -77,9 +92,9 @@ export const getWebsiteLastVisitedEvidence = async (website) => {
             }
           }
         }
-
+      }
     }
-    return result
+    return result;
   } catch (error) {
     return {};
   }