Merge pull request #33 from briehl/develop

Add expires endpoint.

Author: briehl

.travis.yml

@@ -25,6 +25,7 @@ script:
   - export MONGOD=`pwd`/$MONGODB_VER/bin/mongod
   - sed -i "s#^mongo-exe.*#mongo-exe=$MONGOD#" test/test.cfg
   - cat test/test.cfg
+  - make
   - make test
 
 after_script:

Makefile

@@ -1,3 +1,9 @@
+GITCOMMIT = $(shell git rev-parse HEAD)
+
+all:
+	echo "# Don't check this file into git please" > feeds/gitcommit.py
+	echo 'commit = "$(GITCOMMIT)"' >> feeds/gitcommit.py
+
 install:
 	pip install -r requirements.txt
 	pip install -r dev-requirements.txt

README.md

@@ -275,11 +275,32 @@ Takes a list of notifications and marks them as unseen for the user who submitte
 
 ### Expire a notification right away
 This effectively deletes notifications by pushing their expiration time up to the time this request is made.
-* Path: `/api/V1/notification/expire`
+* Path: `/api/V1/notifications/expire`
 * Method: `POST`
-* Required header: `Authorization` - requires a service token.
-* URL Parameters: one of the following:
-    * `id` = the notification id
-    * `external_key` = the external key
-If both parameters are present, a 400 Bad Request error will be returned.
-Also note that services can only expire their own notifications.
+* Required header: `Authorization` - requires a service token or admin token.
+* Expected body:
+```
+{
+    "note_ids": [ list of notification ids ],
+    "external_keys": [ list of external keys ]
+}
+```
+At least one of the above keys must be present. If external keys are used, this must be called by a service
+with a valid service token that maps to the source key in the notification. That is, only services can expire
+their own notifications.
+
+(For now, admins can only expire global notifications)
+* Returns:
+```
+{
+    "expired": {
+        "note_ids": [ list of expired notification ids ],
+        "external_keys": [ list of expired notifications by external key ]
+    },
+    "unauthorized": {
+        "note_ids": [ list of not-expired note ids ],
+        "external_keys": [ list of not-expired external keys ]
+    }
+}
+```
+This will include all of the ids passed to the endpoint, put into one category or the other. Any that were "unauthorized" either don't exist, or came from a different service than the given auth token.

feeds/api/api_v1.py

@@ -194,7 +194,6 @@ def mark_notifications_unseen():
     raise an error.
     Any of these ids that are global, do nothing... for now.
     """
-
     user_id = validate_user_token(get_auth_token(request))
 
     params = _get_mark_notification_params(json.loads(request.get_data()))
@@ -224,7 +223,6 @@ def mark_notifications_seen():
     raise an error.
     Any of these ids that are global, do nothing... for now.
     """
-
     user_id = validate_user_token(get_auth_token(request))
 
     params = _get_mark_notification_params(json.loads(request.get_data()))
@@ -245,6 +243,75 @@ def mark_notifications_seen():
                            'unauthorized_notes': unauthorized_notes}), 200)
 
 
+@api_v1.route('/notifications/expire', methods=['POST'])
+@cross_origin()
+def expire_notifications():
+    """
+    Notifications can be forced to expire (set their expiration time to now).
+    This can be done by:
+        * The service who created the notification
+        * An admin
+    Expects JSON in the body formatted like this:
+    {
+        "note_ids": [notification ids],
+        "external_keys": [keys]
+    }
+    These keys are both optional, but at least one must be present. Any combination of external
+    keys or ids is acceptable, even if they're the same notification.
+    This returns the following:
+    {
+        "expired": {
+            "note_ids": [],
+            "external_keys": []
+        }
+        "unauthorized": {
+            "note_ids": [],
+            "external_keys": []
+        }
+    }
+    This should just return the same lists of values that were input, just shuffled to
+    their final status.
+    """
+    token = get_auth_token(request)
+    is_admin = is_feeds_admin(token)
+    service = None
+    try:
+        service = validate_service_token(token)
+    except InvalidTokenError:
+        if not is_admin:
+            raise InvalidTokenError('Auth token must be either a Service token '
+                                    'or from a user with the FEEDS_ADMIN role!')
+    data = _get_expire_notifications_params(json.loads(request.get_data()))
+    manager = NotificationManager()
+    result = manager.expire_notifications(data.get('note_ids', []), data.get('external_keys', []),
+                                          source=service, is_admin=is_admin)
+    return (flask.jsonify(result), 200)
+
+
+def _get_expire_notifications_params(params):
+    if not isinstance(params, dict):
+        raise IllegalParameterError('Expected a JSON object as an input.')
+
+    if 'note_ids' not in params and 'external_keys' not in params:
+        raise MissingParameterError('Missing parameter "note_ids" or "external_keys"')
+
+    if not isinstance(params.get('note_ids', []), list):
+        raise IllegalParameterError('Expected note_ids to be a list.')
+    else:
+        for i in params.get('note_ids', []):
+            if not isinstance(i, str):
+                raise IllegalParameterError('note_ids must be a list of strings')
+
+    if not isinstance(params.get('external_keys', []), list):
+        raise IllegalParameterError('Expected external_keys to be a list.')
+    else:
+        for i in params.get('external_keys', []):
+            if not isinstance(i, str):
+                raise IllegalParameterError('external_keys must be a list of strings')
+
+    return params
+
+
 def _get_mark_notification_params(params):
     if not isinstance(params, dict):
         raise IllegalParameterError('Expected a JSON object as an input.')

feeds/managers/notification_manager.py

@@ -58,3 +58,43 @@ def get_target_users(self, note: Notification) -> List[str]:
             user_list = list(set(note.users + note.target))
 
         return user_list
+
+    def expire_notifications(self, note_ids: list, external_keys: list, source: str=None, is_admin: bool=False):
+        """
+        Expires notifications.
+        All notifications identified by either their id, or external key, must come from the same
+        source as in the source parameter (i.e. the 'source' key in the database must == the source).
+        Or, an admin can expire any notification.
+        """
+        # Get the notifications from the note_ids and external_keys.
+        # -- mark as unauthorized the ones that don't exist.
+        # If is_admin, cool, expire them all.
+        # If source is not None and not is_admin, make a list of the ones that can be expired (from that source)
+        # expire the ones that we should.
+        # return the results.
+
+        storage = MongoActivityStorage()
+
+        notes_from_id = storage.get_by_id(note_ids, source=source)
+        notes_from_ext_key = {}
+        if source is not None and len(external_keys):
+            notes_from_ext_key = storage.get_by_external_key(external_keys, source)
+        unauthorized = {
+            "note_ids": [k for k in notes_from_id if notes_from_id[k] is None],
+            "external_keys": [k for k in notes_from_ext_key if notes_from_ext_key[k] is None]
+        }
+        ids_to_expire = list()
+        expired = {"note_ids": [], "external_keys": []}
+        for k,v in notes_from_id.items():
+            if v is not None:
+                ids_to_expire.append(k)
+                expired["note_ids"].append(k)
+        for k,v in notes_from_ext_key.items():
+            if v is not None:
+                ids_to_expire.append(v['id'])
+                expired["external_keys"].append(v['external_key'])
+        storage.expire_notifications(ids_to_expire)
+        return {
+            "unauthorized": unauthorized,
+            "expired": expired
+        }

feeds/server.py

@@ -33,7 +33,14 @@
     log_error
 )
 
-VERSION = "0.0.1"
+VERSION = "0.0.2"
+
+try:
+    from feeds import gitcommit
+except ImportError:  # pragma: no cover
+    # tested manually
+    raise ValueError('Did not find git commit file at feeds/gitcommit.py '   # pragma: no cover
+                     'The build may not have completed correctly.')          # pragma: no cover
 
 
 def _initialize_logging():
@@ -92,6 +99,7 @@ def postprocess_request(response):
     def root():
         return flask.jsonify({
             "service": "Notification Feeds Service",
+            "gitcommithash": gitcommit.commit,
             "version": VERSION,
             "servertime": epoch_ms()
         })

feeds/storage/mongodb/activity_storage.py

@@ -5,6 +5,7 @@
     ActivityStorageError
 )
 from pymongo.errors import PyMongoError
+from feeds.util import epoch_ms
 
 
 class MongoActivityStorage(ActivityStorage):
@@ -23,12 +24,6 @@ def add_to_storage(self, activity, target_users: List[str]):
         except PyMongoError as e:
             raise ActivityStorageError("Failed to store activity: " + str(e))
 
-    def get_from_storage(self, activity_ids, user):
-        pass
-
-    def remove_from_storage(self, activity_ids):
-        raise NotImplementedError()
-
     def set_unseen(self, act_ids: List[str], user: str):
         """
         Setting unseen means adding the user to the list of unseens. But we should only do that for
@@ -58,3 +53,53 @@ def set_seen(self, act_ids: List[str], user: str):
         }, {
             '$pull': {'unseen': user}
         })
+
+    def get_by_id(self, act_ids: List[str], source: str=None):
+        """
+        If source is not None, return only those that match the source.
+        Returns a dict mapping from note id to note
+        """
+        if len(act_ids) == 0:
+            return {}
+        coll = get_feeds_collection()
+        query = {
+            'id': {'$in': act_ids}
+        }
+        if source is not None:
+            query['source'] = source
+        notes = {k:None for k in act_ids}
+        curs = coll.find(query)
+        for d in curs:
+            notes[d["id"]] = d
+        return notes
+
+    def get_by_external_key(self, external_keys: List[str], source):
+        """
+        Source HAS to exist here, it's part of the index.
+        Returns a dict mapping from external_key to note
+        """
+        assert source is not None
+        if len(external_keys) == 0:
+            return {}
+        coll = get_feeds_collection()
+        query = {
+            'external_key': {'$in': external_keys},
+            'source': source
+        }
+        notes = {k:None for k in external_keys}
+        curs = coll.find(query)
+        for d in curs:
+            notes[d["external_key"]] = d
+        return notes
+
+    def expire_notifications(self, act_ids: List[str]):
+        """
+        Expires notifications by changing their expiration time to now.
+        """
+        now = epoch_ms()
+        coll = get_feeds_collection()
+        coll.update_many({
+            'id': {'$in': act_ids}
+        }, {
+            '$set': {'expires': now}
+        })

feeds/storage/mongodb/timeline_storage.py

@@ -2,7 +2,7 @@
 from ..base import TimelineStorage
 from .connection import get_feeds_collection
 import logging
-
+from feeds.util import epoch_ms
 
 class MongoTimelineStorage(TimelineStorage):
     """
@@ -21,8 +21,10 @@ def get_timeline(self, count=10, include_seen=False, level=None, verb=None, reve
         """
         # TODO: input validation
         coll = get_feeds_collection()
+        now = epoch_ms()
         query = {
-            "users": self.user_id #{"$all": [self.user_id]}
+            "users": self.user_id,
+            "expires": {"$gt": now}
         }
         if not include_seen:
             query['unseen'] = self.user_id
@@ -37,9 +39,6 @@ def get_timeline(self, count=10, include_seen=False, level=None, verb=None, reve
         serial_notes = [note for note in timeline]
         return serial_notes
 
-    def remove_from_timeline(self, activity_ids):
-        raise NotImplementedError()
-
     def get_single_activity_from_timeline(self, note_id):
         coll = get_feeds_collection()
         query = {
@@ -53,4 +52,4 @@ def get_single_activity_from_timeline(self, note_id):
             note_serial['seen'] = False
         else:
             note_serial['seen'] = True
-        return note_serial
+        return note_serial

test/_data/mongo/README.txt

@@ -29,4 +29,6 @@ test_user
     * 6-10 have test_user2
     * 8,9,10 have test_user3
 * external key:
-    * 1,2,3 have external keys
+    * 1,2,3 have external keys
+* expired (shouldn't be returned in a fetch ever):
+    * 11