Merge branch 'master' into community-learning-test

src/_locales/en_US/messages.json

@@ -223,6 +223,14 @@
         "message": "Learn in Private/Incognito windows",
         "description": "Checkbox label on the general settings page"
     },
+    "options_community_learning_setting": {
+        "message": "Enable community learning and share data about trackers",
+        "description": "Checkbox label on the general settings page"
+    },
+    "options_community_learning_warning": {
+        "message": "When you enable community learning, your browser will share some information it collects about trackers with EFF. Specifically, each time your instance of Privacy Badger observes a particular tracker on a website that it has not seen before, it will share the origin (top-level domain +1) of both the tracker and the website, as well as the type of tracking action that it observed. EFF will only use this information for generating community learning lists, and will never share personal information with third parties. For more details, see our privacy policy: https://link.to.come",
+        "description": "Checkbox label on the general settings page"
+    },
     "options_incognito_warning": {
         "message": "Enabling learning in Private/Incognito windows may leave traces of your private browsing history on your computer. By default, Privacy Badger will block trackers it already knows about in Private/Incognito windows, but it won't learn about new trackers. You might want to enable this option if a lot of your browsing happens in Private/Incognito windows.",
         "description": "Tooltip on the general settings page"

src/js/background.js

@@ -830,6 +830,7 @@ Badger.prototype = {
     preventWebRTCIPLeak: false,
     seenComic: false,
     sendDNTSignal: true,
+    shareLearning: false,
     showCounter: true,
     showExpandedTrackingSection: false,
     showIntroPage: true,
@@ -1087,13 +1088,36 @@ Badger.prototype = {
    * and if tab_id is for an incognito window,
    * is learning in incognito windows enabled?
    */
-  isLearningEnabled(tab_id) {
+  isLocalLearningEnabled(tab_id) {
     return (
       this.getSettings().getItem("learnLocally") &&
       incognito.learningEnabled(tab_id)
     );
   },
 
+  /**
+   * Is community learning generally enabled,
+   * and is tab_id in a regular (not incognito) window?
+   */
+  isCommunityLearningEnabled(tab_id) {
+    return (
+      this.getSettings().getItem("shareLearning") &&
+      !incognito.isIncognito(tab_id)
+    );
+  },
+
+  /**
+   * Is any kind of learning (local or community) enabled on this tab?
+   *
+   * TODO: should community learning happen in incognito tabs?
+   */
+  isLearningEnabled(tab_id) {
+    return (
+      this.isLocalLearningEnabled(tab_id) ||
+      this.isCommunityLearningEnabled(tab_id)
+    );
+  },
+
   isDNTSignalEnabled: function() {
     return this.getSettings().getItem("sendDNTSignal");
   },
@@ -1148,7 +1172,7 @@ Badger.prototype = {
   },
 
   /**
-   * Checks if local storage ( in dict) has any high-entropy keys
+   * Checks if local storage (in dict) has any high-entropy keys
    *
    * @param {Object} lsItems Local storage dict
    * @returns {boolean} true if it seems there are supercookies

src/js/constants.js

@@ -41,6 +41,17 @@ var exports = {
   TRACKING_THRESHOLD: 3,
   MAX_COOKIE_ENTROPY: 12,
 
+  // The max amount of time (in milliseconds) that PB will wait before sharing a
+  // tracking action with EFF for community learning
+  MAX_CL_WAIT_TIME: 5 * 60 * 1000, // five minutes
+
+  // The probability that any given tracking action will be logged to the
+  // community server, as a float from 0.0 to 1.0
+  CL_PROBABILITY: 1.0,
+
+  // size of the in-memory community learning cache
+  CL_CACHE_SIZE: 5000,
+
   DNT_POLICY_CHECK_INTERVAL: 1000, // one second
 };
 
@@ -51,5 +62,12 @@ exports.BLOCKED_ACTIONS = new Set([
   exports.USER_COOKIEBLOCK,
 ]);
 
+exports.TRACKER_TYPES = Object.freeze({
+  COOKIE: "cookie",
+  COOKIE_SHARE: "cookie_share",
+  SUPERCOOKIE: "supercookie",
+  FINGERPRINT: "fingerprint",
+})
+
 return exports;
 })();

src/js/heuristicblocking.js

@@ -37,6 +37,9 @@ function HeuristicBlocker(pbStorage) {
   // impossible to attribute to a tab.
   this.tabOrigins = {};
   this.tabUrls = {};
+
+  // in-memory cache for community learning
+  this.previouslySharedTrackers = new Set();
 }
 
 HeuristicBlocker.prototype = {
@@ -105,8 +108,10 @@ HeuristicBlocker.prototype = {
    */
   // TODO more like heuristicLearningFromCookies ... check DESIGN doc
   heuristicBlockingAccounting: function (details, check_for_cookie_share) {
+    let tab_id = details.tabId;
+
     // ignore requests that are outside a tabbed window
-    if (details.tabId < 0 || !badger.isLearningEnabled(details.tabId)) {
+    if (tab_id < 0 || !badger.isLearningEnabled(tab_id)) {
       return {};
     }
 
@@ -115,12 +120,12 @@ HeuristicBlocker.prototype = {
 
     // if this is a main window request, update tab data and quit
     if (details.type == "main_frame") {
-      self.tabOrigins[details.tabId] = window.getBaseDomain(request_host);
-      self.tabUrls[details.tabId] = details.url;
+      self.tabOrigins[tab_id] = window.getBaseDomain(request_host);
+      self.tabUrls[tab_id] = details.url;
       return {};
     }
 
-    let tab_base = self.tabOrigins[details.tabId];
+    let tab_base = self.tabOrigins[tab_id];
     if (!tab_base) {
       return {};
     }
@@ -152,15 +157,15 @@ HeuristicBlocker.prototype = {
 
     // check if there are tracking cookies
     if (hasCookieTracking(details)) {
-      self._recordPrevalence(request_host, request_base, tab_base);
+      self._recordPrevalence(request_host, request_base, tab_base, tab_id, constants.TRACKER_TYPES.COOKIE);
       return {};
     }
 
     // check for cookie sharing iff this is an image in the top-level frame, and the request URL has parameters
     if (check_for_cookie_share && details.type == 'image' && details.frameId === 0 && details.url.indexOf('?') > -1) {
       // get all non-HttpOnly cookies for the top-level frame
       // and pass those to the cookie-share accounting function
-      let tab_url = self.tabUrls[details.tabId];
+      let tab_url = self.tabUrls[tab_id];
 
       let config = {
         url: tab_url
@@ -174,7 +179,7 @@ HeuristicBlocker.prototype = {
         if (cookies.length >= 1) {
           // TODO refactor with new URI() above?
           let searchParams = (new URL(details.url)).searchParams;
-          self.pixelCookieShareAccounting(tab_url, tab_base, searchParams, request_host, request_base, cookies);
+          self.pixelCookieShareAccounting(tab_id, tab_url, tab_base, searchParams, request_host, request_base, cookies);
         }
       });
     }
@@ -192,7 +197,7 @@ HeuristicBlocker.prototype = {
    * Doesn't catch cookie syncing (3rd party -> 3rd party),
    * but most of those tracking cookies should be blocked anyway.
    */
-  pixelCookieShareAccounting: function (tab_url, tab_base, searchParams, request_host, request_base, cookies) {
+  pixelCookieShareAccounting: function (tab_id, tab_url, tab_base, searchParams, request_host, request_base, cookies) {
     const TRACKER_ENTROPY_THRESHOLD = 33,
       MIN_STR_LEN = 8;
 
@@ -263,7 +268,7 @@ HeuristicBlocker.prototype = {
             log("Found high-entropy cookie share from", tab_base, "to", request_host,
               ":", entropy, "bits\n  cookie:", cookie.name, '=', cookie.value,
               "\n  arg:", key, "=", value, "\n  substring:", s);
-            this._recordPrevalence(request_host, request_base, tab_base);
+            this._recordPrevalence(request_host, request_base, tab_base, tab_id, constants.TRACKER_TYPES.COOKIE_SHARE);
             return;
           }
         }
@@ -277,8 +282,10 @@ HeuristicBlocker.prototype = {
    * @param {String} tracker_fqdn The fully qualified domain name of the tracker
    * @param {String} tracker_base Base domain of the third party tracker
    * @param {String} site_base Base domain of page where tracking occurred
+   * @param {Integer} tab_id the ID of the tab the user is in
+   * @param {String} tracker_type the kind of tracking action that was observed
    */
-  updateTrackerPrevalence: function (tracker_fqdn, tracker_base, site_base) {
+  updateTrackerPrevalence: function (tracker_fqdn, tracker_base, site_base, tab_id, tracker_type) {
     // abort if we already made a decision for this fqdn
     let action = this.storage.getAction(tracker_fqdn);
     if (action != constants.NO_TRACKING && action != constants.ALLOW) {
@@ -288,7 +295,9 @@ HeuristicBlocker.prototype = {
     this._recordPrevalence(
       tracker_fqdn,
       tracker_base,
-      site_base
+      site_base,
+      tab_id,
+      tracker_type
     );
   },
 
@@ -304,8 +313,10 @@ HeuristicBlocker.prototype = {
    * @param {String} tracker_fqdn The FQDN of the third party tracker
    * @param {String} tracker_base Base domain of the third party tracker
    * @param {String} site_base Base domain of page where tracking occurred
+   * @param {Integer} tab_id the ID of the tab the user is in
+   * @param {String} tracker_type the kind of tracking action that was observed
    */
-  _recordPrevalence: function (tracker_fqdn, tracker_base, site_base) {
+  _recordPrevalence: function (tracker_fqdn, tracker_base, site_base, tab_id, tracker_type) {
     // GDPR Consent Management Provider
     // https://github.com/EFForg/privacybadger/pull/2245#issuecomment-545545717
     if (tracker_base == "consensu.org") {
@@ -330,22 +341,81 @@ HeuristicBlocker.prototype = {
       return;
     }
 
+    // If community learning is enabled, queue up a request to the EFF server
+    if (badger.isCommunityLearningEnabled(tab_id)) {
+      let page_fqdn = (new URI(this.tabUrls[tab_id])).host;
+      self.shareTrackerInfo(page_fqdn, tracker_fqdn, tracker_type);
+    }
+
+    // If local learning is enabled,
     // record that we've seen this tracker on this domain
-    firstParties.push(site_base);
-    snitchMap.setItem(tracker_base, firstParties);
-
-    // ALLOW indicates this is a tracker still below TRACKING_THRESHOLD
-    // (vs. NO_TRACKING for resources we haven't seen perform tracking yet).
-    // see https://github.com/EFForg/privacybadger/pull/1145#discussion_r96676710
-    self.storage.setupHeuristicAction(tracker_fqdn, constants.ALLOW);
-    self.storage.setupHeuristicAction(tracker_base, constants.ALLOW);
-
-    // (cookie)block the tracker if it has been seen on multiple first party domains
-    if (firstParties.length >= constants.TRACKING_THRESHOLD) {
-      log("blocklisting", tracker_fqdn);
-      self.blocklistOrigin(tracker_base, tracker_fqdn);
+    if (badger.isLocalLearningEnabled(tab_id)) {
+      firstParties.push(site_base);
+      snitchMap.setItem(tracker_base, firstParties);
+
+      // ALLOW indicates this is a tracker still below TRACKING_THRESHOLD
+      // (vs. NO_TRACKING for resources we haven't seen perform tracking yet).
+      // see https://github.com/EFForg/privacybadger/pull/1145#discussion_r96676710
+      self.storage.setupHeuristicAction(tracker_fqdn, constants.ALLOW);
+      self.storage.setupHeuristicAction(tracker_base, constants.ALLOW);
+
+      // (cookie)block the tracker if it has been seen on multiple first party domains
+      if (firstParties.length >= constants.TRACKING_THRESHOLD) {
+        log("blocklisting", tracker_fqdn);
+        self.blocklistOrigin(tracker_base, tracker_fqdn);
+      }
+    }
+  },
+
+  /**
+   * Share information about a tracker for community learning
+   */
+  shareTrackerInfo: function(page_host, tracker_host, tracker_type) {
+    // Share a random sample of trackers we observe
+    if (Math.random() < constants.CL_PROBABILITY) {
+      // check if we've shared this tracker recently
+      // note that this check comes after checking against the snitch map
+      let tr_str = page_host + '+' + tracker_host + '+' + tracker_type;
+      if (this.previouslySharedTrackers.has(tr_str)) {
+        return;
+      }
+
+      // add this entry to the cache
+      this.previouslySharedTrackers.add(tr_str);
+
+      // if the cache gets too big, cut it in half
+      if (this.previouslySharedTrackers.size > constants.CL_CACHE_SIZE) {
+        this.previouslySharedTrackers = new Set(
+          // An array created from the set will have all of its entries ordered
+          // by when they were added
+          Array.from(this.previouslySharedTrackers).slice(
+            // keep the most recent half of the cache entries
+            Math.floor(constants.CL_CACHE_SIZE / 2)
+          )
+        );
+      }
+
+      // now make the request to the database server
+      setTimeout(function () {
+        fetch("http://localhost:8080", {
+          method: "POST",
+          body: JSON.stringify({
+            tracker_data: {
+              page_host: page_host,
+              tracker_host: tracker_host,
+              tracker_type: tracker_type,
+            }
+          })
+        }).then(res => {
+          if (!res.ok) {
+            console.log("tracking action logging failed:", res);
+          }
+        });
+      // share info after a random delay, to reduce network load on browser
+      }, Math.floor(Math.random() * constants.MAX_CL_WAIT_TIME));
     }
   }
+
 };
 
 

src/js/incognito.js

@@ -25,23 +25,30 @@ function startListeners() {
   chrome.tabs.onRemoved.addListener(onRemovedListener);
 }
 
+function isIncognito(tab_id) {
+  // if we don't have incognito data for whatever reason,
+  // default to "true"
+  if (!tabs.hasOwnProperty(tab_id)) {
+    return true;
+  }
+  // else, do not learn in incognito tabs
+  return tabs[tab_id];
+}
+
 function learningEnabled(tab_id) {
   if (badger.getSettings().getItem("learnInIncognito")) {
     // treat all pages as if they're not incognito
     return true;
   }
-  // if we don't have incognito data for whatever reason,
-  // default to disabled
-  if (!tabs.hasOwnProperty(tab_id)) {
-    return false;
-  }
-  // else, do not learn in incognito tabs
-  return !tabs[tab_id];
+
+  // otherwise, return true if this tab is _not_ incognito
+  return !isIncognito(tab_id);
 }
 
 /************************************** exports */
 let exports = {
   learningEnabled,
+  isIncognito,
   startListeners,
 };
 return exports;

src/js/options.js

@@ -224,6 +224,18 @@ function loadOptions() {
       });
     });
 
+  $('#community-learning-checkbox')
+    .prop("checked", OPTIONS_DATA.settings.shareLearning)
+    .on("click", (event) => {
+      const enabled = $(event.currentTarget).prop("checked");
+      chrome.runtime.sendMessage({
+        type: "updateSettings",
+        data: {
+          shareLearning: enabled
+        }
+      }, function () {});
+    });
+
   $('#show-nontracking-domains-checkbox')
     .prop("disabled", OPTIONS_DATA.settings.learnLocally ? false : "disabled")
     .prop("checked", (

src/js/storage.js

@@ -622,9 +622,25 @@ BadgerStorage.prototype = {
     } else if (self.name == "snitch_map") {
       for (let tracker_base in mapData) {
         let siteBases = mapData[tracker_base];
-        for (let siteBase of siteBases) {
-          badger.heuristicBlocking.updateTrackerPrevalence(
-            tracker_base, tracker_base, siteBase);
+
+        let firstParties = [];
+        if (self.hasItem(tracker_base)) {
+          firstParties = self.getItem(tracker_base);
+        }
+
+        // this uses the same logic as _recordPrevalence(), but ignores
+        // checks for local learning and community learning
+        for (let site_base of siteBases) {
+          firstParties.push(site_base);
+          self.setItem(tracker_base, firstParties);
+
+          badger.storage.setupHeuristicAction(tracker_base, constants.ALLOW);
+
+          // block the origin if it has been seen on multiple first party domains
+          if (firstParties.length >= constants.TRACKING_THRESHOLD) {
+            log("blocklisting", tracker_base);
+            badger.heuristicBlocking.blocklistOrigin(tracker_base, tracker_base);
+          }
         }
       }
     }