[Core-551] Client: Add methods to test read/write/ownership of catalog objects (#12711)

GitOrigin-RevId: bbadc84268bf6fdd67455fda2fad30aa8120e7ea

Author: stephencpope

descarteslabs/auth/auth.py

@@ -153,6 +153,28 @@ class Auth:
     KEY_JWT_TOKEN = "jwt_token"
     KEY_ALT_JWT_TOKEN = "JWT_TOKEN"
 
+    # The various prefixes that can be used in Catalog ACLs.
+    ACL_PREFIX_USER = "user:"  # Followed by the user's sha1 hash
+    ACL_PREFIX_EMAIL = "email:"  # Followed by the user's email
+    ACL_PREFIX_GROUP = "group:"  # Followed by a lowercase group
+    ACL_PREFIX_ORG = "org:"  # Followed by a lowercase org name
+    ACL_PREFIX_ACCESS = "access-id:"  # Followed by the purchase-specific access id
+    # Note that the access-id, including the prefix `access_id:`, is matched against
+    # a group with the same name. In other words `group:access-id:<access-id>` will
+    # match against `access-id:<access-id>` (assuming the `<access_id>` is identical).
+
+    # these match the values in descarteslabs/common/services/python_auth/groups.py
+    ORG_ADMIN_SUFFIX = ":org-admin"
+    RESOURCE_ADMIN_SUFFIX = ":resource-admin"
+
+    # These are cache keys for caching various data in the object's __dict__.
+    # These are scrubbed out with `_clear_cache()` when retrieving a new token.
+    KEY_PAYLOAD = "_payload"
+    KEY_ALL_ACL_SUBJECTS = "_aas"
+    KEY_ALL_ACL_SUBJECTS_AS_SET = "_aasas"
+    KEY_ALL_OWNER_ACL_SUBJECTS = "_aoas"
+    KEY_ALL_OWNER_ACL_SUBJECTS_AS_SET = "_aoasas"
+
     __attrs__ = [
         "domain",
         "scope",
@@ -585,7 +607,13 @@ def payload(self):
         OauthError
             Raised when a token cannot be obtained or refreshed.
         """
-        return self._get_payload(self.token)
+        payload = self.__dict__.get(self.KEY_PAYLOAD)
+
+        if payload is None:
+            payload = self._get_payload(self.token)
+            self.__dict__[self.KEY_PAYLOAD] = payload
+
+        return payload
 
     @staticmethod
     def _get_payload(token):
@@ -754,6 +782,9 @@ def _get_token(self, timeout=100):
         else:
             raise OauthError("Could not retrieve token")
 
+        # clear out payload and subjects cache
+        self._clear_cache()
+
         token_info = {}
 
         # Read the token from the token_info_path, and save it again
@@ -797,6 +828,121 @@ def namespace(self):
             self._namespace = sha1(self.payload["sub"].encode("utf-8")).hexdigest()
         return self._namespace
 
+    @property
+    def all_acl_subjects(self):
+        """
+        A list of all ACL subjects identifying this user (the user itself, the org, the
+        groups) which can be used in ACL queries.
+        """
+        subjects = self.__dict__.get(self.KEY_ALL_ACL_SUBJECTS)
+
+        if subjects is None:
+            subjects = [self.ACL_PREFIX_USER + self.namespace]
+
+            if email := self.payload.get("email"):
+                subjects.append(self.ACL_PREFIX_EMAIL + email.lower())
+
+            if org := self.payload.get("org"):
+                subjects.append(self.ACL_PREFIX_ORG + org)
+
+            subjects += [
+                self.ACL_PREFIX_GROUP + group for group in self._active_groups()
+            ]
+            self.__dict__[self.KEY_ALL_ACL_SUBJECTS] = subjects
+
+        return subjects
+
+    @property
+    def all_acl_subjects_as_set(self):
+        subjects_as_set = self.__dict__.get(self.KEY_ALL_ACL_SUBJECTS_AS_SET)
+
+        if subjects_as_set is None:
+            subjects_as_set = set(self.all_acl_subjects)
+            self.__dict__[self.KEY_ALL_ACL_SUBJECTS_AS_SET] = subjects_as_set
+
+        return subjects_as_set
+
+    @property
+    def all_owner_acl_subjects(self):
+        """
+        A list of ACL subjects identifying this user (the user itself, the org,
+        org admin and catalog admins) which can be used in owner ACL queries.
+        """
+        subjects = self.__dict__.get(self.KEY_ALL_OWNER_ACL_SUBJECTS)
+
+        if subjects is None:
+            subjects = [self.ACL_PREFIX_USER + self.namespace]
+
+            subjects.extend(
+                [self.ACL_PREFIX_ORG + org for org in self.get_org_admins() if org]
+            )
+            subjects.extend(
+                [
+                    self.ACL_PREFIX_ACCESS + access_id
+                    for access_id in self.get_resource_admins()
+                    if access_id
+                ]
+            )
+            self.__dict__[self.KEY_ALL_OWNER_ACL_SUBJECTS] = subjects
+
+        return subjects
+
+    @property
+    def all_owner_acl_subjects_as_set(self):
+        subjects_as_set = self.__dict__.get(self.KEY_ALL_OWNER_ACL_SUBJECTS_AS_SET)
+
+        if subjects_as_set is None:
+            subjects_as_set = set(self.all_owner_acl_subjects)
+            self.__dict__[self.KEY_ALL_OWNER_ACL_SUBJECTS_AS_SET] = subjects_as_set
+
+        return subjects_as_set
+
+    def get_org_admins(self):
+        # This retrieves the value of the org to be added if the user has one or
+        # more org-admin groups, otherwise the empty list.
+        return [
+            group[: -len(self.ORG_ADMIN_SUFFIX)]
+            for group in self.payload.get("groups", [])
+            if group.endswith(self.ORG_ADMIN_SUFFIX)
+        ]
+
+    def get_resource_admins(self):
+        # This retrieves the value of the access-id to be added if the user has one or
+        # more resource-admin groups, otherwise the empty list.
+        return [
+            group[: -len(self.RESOURCE_ADMIN_SUFFIX)]
+            for group in self.payload.get("groups", [])
+            if group.endswith(self.RESOURCE_ADMIN_SUFFIX)
+        ]
+
+    def _active_groups(self):
+        """
+        Attempts to filter groups to just the ones that are currently valid for this
+        user.  If they have a colon, the prefix leading up to the colon must be the
+        user's current org, otherwise the user should not actually have rights with
+        this group.
+        """
+        org = self.payload.get("org")
+        for group in self.payload.get("groups", []):
+            parts = group.split(":")
+
+            if len(parts) == 1:
+                yield group
+            elif org and parts[0] == org:
+                yield group
+
+    def _clear_cache(self):
+        for key in (
+            self.KEY_PAYLOAD,
+            self.KEY_ALL_ACL_SUBJECTS,
+            self.KEY_ALL_ACL_SUBJECTS_AS_SET,
+            self.KEY_ALL_OWNER_ACL_SUBJECTS,
+            self.KEY_ALL_OWNER_ACL_SUBJECTS_AS_SET,
+        ):
+            if key in self.__dict__:
+                del self.__dict__[key]
+        self._namespace = None
+
     def __getstate__(self):
         return dict((attr, getattr(self, attr)) for attr in self.__attrs__)
 

descarteslabs/auth/tests/test_auth.py

@@ -490,6 +490,68 @@ def test_domain(self):
         a = Auth()
         assert a.domain == domain
 
+    def test_all_acl_subjects(self):
+        auth = Auth(
+            client_secret="client_secret",
+            client_id="ZOBAi4UROl5gKZIpxxlwOEfx8KpqXf2c",
+        )
+        token = b".".join(
+            (
+                base64.b64encode(to_bytes(p))
+                for p in [
+                    "header",
+                    json.dumps(
+                        dict(
+                            sub="some|user",
+                            groups=["public"],
+                            org="some-org",
+                            exp=9999999999,
+                            aud="ZOBAi4UROl5gKZIpxxlwOEfx8KpqXf2c",
+                        )
+                    ),
+                    "sig",
+                ]
+            )
+        )
+        auth._token = token
+
+        assert {
+            Auth.ACL_PREFIX_USER + auth.namespace,
+            f"{Auth.ACL_PREFIX_GROUP}public",
+            f"{Auth.ACL_PREFIX_ORG}some-org",
+        } == set(auth.all_acl_subjects)
+
+    def test_all_acl_subjects_ignores_bad_org_groups(self):
+        auth = Auth(
+            client_secret="client_secret",
+            client_id="ZOBAi4UROl5gKZIpxxlwOEfx8KpqXf2c",
+        )
+        token = b".".join(
+            (
+                base64.b64encode(to_bytes(p))
+                for p in [
+                    "header",
+                    json.dumps(
+                        dict(
+                            sub="some|user",
+                            groups=["public", "some-org:baz", "other:baz"],
+                            org="some-org",
+                            exp=9999999999,
+                            aud="ZOBAi4UROl5gKZIpxxlwOEfx8KpqXf2c",
+                        )
+                    ),
+                    "sig",
+                ]
+            )
+        )
+        auth._token = token
+        assert {
+            Auth.ACL_PREFIX_USER + auth.namespace,
+            f"{Auth.ACL_PREFIX_ORG}some-org",
+            f"{Auth.ACL_PREFIX_GROUP}public",
+            f"{Auth.ACL_PREFIX_GROUP}some-org:baz",
+        } == set(auth.all_acl_subjects)
+
 
 if __name__ == "__main__":
     unittest.main()

descarteslabs/core/catalog/__init__.py

@@ -90,6 +90,7 @@
     SummarySearchMixin,
 )
 from .catalog_base import (
+    AuthCatalogObject,
     CatalogClient,
     CatalogObject,
     DeletedObjectError,
@@ -112,6 +113,7 @@
 __all__ = [
     "AggregateDateField",
     "AttributeValidationError",
+    "AuthCatalogObject",
     "Band",
     "BandCollection",
     "BandType",

descarteslabs/core/catalog/blob.py

@@ -25,17 +25,18 @@
     DocumentState,
     EnumAttribute,
     GeometryAttribute,
-    ListAttribute,
     StorageState,
     Timestamp,
     TypedAttribute,
     parse_iso_datetime,
 )
 from .blob_download import BlobDownload
 from .catalog_base import (
+    AuthCatalogObject,
     CatalogClient,
-    CatalogObject,
     check_deleted,
+    check_derived,
+    hybridmethod,
     UnsavedObjectError,
 )
 from .search import AggregateDateField, GeoSearch, SummarySearchMixin
@@ -118,7 +119,7 @@ class BlobSearch(SummarySearchMixin, GeoSearch):
     DEFAULT_AGGREGATE_DATE_FIELD = AggregateDateField.CREATED
 
 
-class Blob(CatalogObject):
+class Blob(AuthCatalogObject):
     """A stored blob (arbitrary bytes) that can be searched and retrieved.
 
     Instantiating a blob indicates that you want to create a *new* Descartes Labs
@@ -139,35 +140,6 @@ class Blob(CatalogObject):
         the exception of properties (`ATTRIBUTES`, `is_modified`, and `state`), any
         attribute listed below can also be used as a keyword argument.  Also see
         `~Blob.ATTRIBUTES`.
-
-
-    .. _blob_note:
-
-    Note
-    ----
-    The ``reader`` and ``writer`` IDs must be prefixed with ``email:``, ``user:``,
-    ``group:`` or ``org:``.  The ``owner`` ID only accepts ``org:`` and ``user:``.
-    Using ``org:`` as an ``owner`` will assign those privileges only to administrators
-    for that organization; using ``org:`` as a ``reader`` or ``writer`` assigns those
-    privileges to everyone in that organization.  The `readers` and `writers` attributes
-    are only visible in full to the `owners`. If you are a `reader` or a `writer` those
-    attributes will only display the element of those lists by which you are gaining
-    read or write access.
-
-    Any user with ``owner`` privileges is able to read the blob attributes or data,
-    modify the blob attributes, or delete the blob, including reading and modifying the
-    ``owners``, ``writers``, and ``readers`` attributes.
-
-    Any user with ``writer`` privileges is able to read the blob attributes or data,
-    or modify the blob attributes, but not delete the blob. A ``writer`` can read the
-    ``owners`` and can only read the entry in the ``writers`` and/or ``readers``
-    by which they gain access to the blob.
-
-    Any user with ``reader`` privileges is able to read the blob attributes or data.
-    A ``reader`` can read the ``owners`` and can only read the entry
-    in the ``writers`` and/or ``readers`` by which they gain access to the blob.
-
-    Also see :doc:`Sharing Resources </guides/sharing>`.
     """
 
     _doc_type = "storage"
@@ -269,33 +241,6 @@ class Blob(CatalogObject):
     hash = TypedAttribute(
         str, doc="""str, optional: Content hash (MD5) for the blob."""
     )
-    owners = ListAttribute(
-        TypedAttribute(str),
-        doc="""list(str), optional: User, group, or organization IDs that own this blob.
-
-        Defaults to [``user:current_user``, ``org:current_org``].  The owner can edit,
-        delete, and change access to this blob.  :ref:`See this note <blob_note>`.
-
-        *Filterable*.
-        """,
-    )
-    readers = ListAttribute(
-        TypedAttribute(str),
-        doc="""list(str), optional: User, email, group, or organization IDs that can read this blob.
-
-        Will be empty by default.  This attribute is only available in full to the `owners`
-        of the blob.  :ref:`See this note <blob_note>`.
-        """,
-    )
-    writers = ListAttribute(
-        TypedAttribute(str),
-        doc="""list(str), optional: User, group, or organization IDs that can edit this blob.
-
-        Writers will also have read permission.  Writers will be empty by default.
-        See note below.  This attribute is only available in full to the `owners` of the blob.
-        :ref:`See this note <blob_note>`.
-        """,
-    )
 
     @classmethod
     def namespace_id(cls, namespace_id, client=None):
@@ -1020,8 +965,9 @@ def _do_download(self, dest=None, range=None):
             finally:
                 r.close()
 
-    @classmethod
-    def _cls_delete(cls, id, client=None):
+    @hybridmethod
+    @check_derived
+    def delete(cls, id, client=None):
         """Delete the catalog object with the given `id`.
 
         Parameters
@@ -1051,6 +997,10 @@ def _cls_delete(cls, id, client=None):
         Example
         -------
         >>> Image.delete('my-image-id') # doctest: +SKIP
+
+        There is also an instance ``delete`` method that can be used to delete a blob.
+        It accepts no parameters and also returns a ``BlobDeletionTaskStatus``. Once
+        deleted, you cannot use the blob and should release any references.
         """
         if client is None:
             client = CatalogClient.get_default_client()
@@ -1062,7 +1012,9 @@ def _cls_delete(cls, id, client=None):
         except NotFoundError:
             return None
 
-    def _instance_delete(self):
+    @delete.instancemethod
+    @check_deleted
+    def delete(self):
         """Delete this catalog object from the Descartes Labs catalog.
 
         Once deleted, you cannot use the catalog object and should release any