Skip to content

Commit

Permalink
feat: support includeFoldersAsPrefixes (#1223)
Browse files Browse the repository at this point in the history
* feat: support includeFoldersAsPrefixes

* sys test

* update sys test with cleanup
  • Loading branch information
cojenco authored Mar 18, 2024
1 parent 3928aa0 commit 7bb8065
Show file tree
Hide file tree
Showing 5 changed files with 66 additions and 0 deletions.
7 changes: 7 additions & 0 deletions google/cloud/storage/bucket.py
Original file line number Diff line number Diff line change
Expand Up @@ -1306,6 +1306,7 @@ def list_blobs(
timeout=_DEFAULT_TIMEOUT,
retry=DEFAULT_RETRY,
match_glob=None,
include_folders_as_prefixes=None,
soft_deleted=None,
):
"""Return an iterator used to find blobs in the bucket.
Expand Down Expand Up @@ -1388,6 +1389,11 @@ def list_blobs(
The string value must be UTF-8 encoded. See:
https://cloud.google.com/storage/docs/json_api/v1/objects/list#list-object-glob
:type include_folders_as_prefixes: bool
(Optional) If true, includes Folders and Managed Folders in the set of
``prefixes`` returned by the query. Only applicable if ``delimiter`` is set to /.
See: https://cloud.google.com/storage/docs/managed-folders
:type soft_deleted: bool
:param soft_deleted:
(Optional) If true, only soft-deleted objects will be listed as distinct results in order of increasing
Expand Down Expand Up @@ -1415,6 +1421,7 @@ def list_blobs(
timeout=timeout,
retry=retry,
match_glob=match_glob,
include_folders_as_prefixes=include_folders_as_prefixes,
soft_deleted=soft_deleted,
)

Expand Down
9 changes: 9 additions & 0 deletions google/cloud/storage/client.py
Original file line number Diff line number Diff line change
Expand Up @@ -1184,6 +1184,7 @@ def list_blobs(
timeout=_DEFAULT_TIMEOUT,
retry=DEFAULT_RETRY,
match_glob=None,
include_folders_as_prefixes=None,
soft_deleted=None,
):
"""Return an iterator used to find blobs in the bucket.
Expand Down Expand Up @@ -1283,6 +1284,11 @@ def list_blobs(
The string value must be UTF-8 encoded. See:
https://cloud.google.com/storage/docs/json_api/v1/objects/list#list-object-glob
include_folders_as_prefixes (bool):
(Optional) If true, includes Folders and Managed Folders in the set of
``prefixes`` returned by the query. Only applicable if ``delimiter`` is set to /.
See: https://cloud.google.com/storage/docs/managed-folders
soft_deleted (bool):
(Optional) If true, only soft-deleted objects will be listed as distinct results in order of increasing
generation number. This parameter can only be used successfully if the bucket has a soft delete policy.
Expand Down Expand Up @@ -1325,6 +1331,9 @@ def list_blobs(
if fields is not None:
extra_params["fields"] = fields

if include_folders_as_prefixes is not None:
extra_params["includeFoldersAsPrefixes"] = include_folders_as_prefixes

if soft_deleted is not None:
extra_params["softDeleted"] = soft_deleted

Expand Down
41 changes: 41 additions & 0 deletions tests/system/test_bucket.py
Original file line number Diff line number Diff line change
Expand Up @@ -653,6 +653,47 @@ def test_bucket_list_blobs_w_match_glob(
assert [blob.name for blob in blobs] == expected_names


def test_bucket_list_blobs_include_managed_folders(
storage_client,
buckets_to_delete,
blobs_to_delete,
hierarchy_filenames,
):
bucket_name = _helpers.unique_name("ubla-mf")
bucket = storage_client.bucket(bucket_name)
bucket.iam_configuration.uniform_bucket_level_access_enabled = True
_helpers.retry_429_503(bucket.create)()
buckets_to_delete.append(bucket)

payload = b"helloworld"
for filename in hierarchy_filenames:
blob = bucket.blob(filename)
blob.upload_from_string(payload)
blobs_to_delete.append(blob)

# Make API call to create a managed folder.
# TODO: change to use storage control client once available.
path = f"/b/{bucket_name}/managedFolders"
properties = {"name": "managedfolder1"}
storage_client._post_resource(path, properties)

expected_prefixes = set(["parent/"])
blob_iter = bucket.list_blobs(delimiter="/")
list(blob_iter)
assert blob_iter.prefixes == expected_prefixes

# Test that managed folders are only included when IncludeFoldersAsPrefixes is set.
expected_prefixes = set(["parent/", "managedfolder1/"])
blob_iter = bucket.list_blobs(delimiter="/", include_folders_as_prefixes=True)
list(blob_iter)
assert blob_iter.prefixes == expected_prefixes

# Cleanup: API call to delete a managed folder.
# TODO: change to use storage control client once available.
path = f"/b/{bucket_name}/managedFolders/managedfolder1"
storage_client._delete_resource(path)


def test_bucket_update_retention_period(
storage_client,
buckets_to_delete,
Expand Down
6 changes: 6 additions & 0 deletions tests/unit/test_bucket.py
Original file line number Diff line number Diff line change
Expand Up @@ -1177,6 +1177,7 @@ def test_list_blobs_w_defaults(self):
expected_versions = None
expected_projection = "noAcl"
expected_fields = None
expected_include_folders_as_prefixes = None
soft_deleted = None
client.list_blobs.assert_called_once_with(
bucket,
Expand All @@ -1193,6 +1194,7 @@ def test_list_blobs_w_defaults(self):
timeout=self._get_default_timeout(),
retry=DEFAULT_RETRY,
match_glob=expected_match_glob,
include_folders_as_prefixes=expected_include_folders_as_prefixes,
soft_deleted=soft_deleted,
)

Expand All @@ -1206,6 +1208,7 @@ def test_list_blobs_w_explicit(self):
start_offset = "c"
end_offset = "g"
include_trailing_delimiter = True
include_folders_as_prefixes = True
versions = True
soft_deleted = True
projection = "full"
Expand All @@ -1231,6 +1234,7 @@ def test_list_blobs_w_explicit(self):
timeout=timeout,
retry=retry,
match_glob=match_glob,
include_folders_as_prefixes=include_folders_as_prefixes,
soft_deleted=soft_deleted,
)

Expand All @@ -1247,6 +1251,7 @@ def test_list_blobs_w_explicit(self):
expected_versions = versions
expected_projection = projection
expected_fields = fields
expected_include_folders_as_prefixes = include_folders_as_prefixes
expected_soft_deleted = soft_deleted
other_client.list_blobs.assert_called_once_with(
bucket,
Expand All @@ -1263,6 +1268,7 @@ def test_list_blobs_w_explicit(self):
timeout=timeout,
retry=retry,
match_glob=expected_match_glob,
include_folders_as_prefixes=expected_include_folders_as_prefixes,
soft_deleted=expected_soft_deleted,
)

Expand Down
3 changes: 3 additions & 0 deletions tests/unit/test_client.py
Original file line number Diff line number Diff line change
Expand Up @@ -2015,6 +2015,7 @@ def test_list_blobs_w_explicit_w_user_project(self):
start_offset = "c"
end_offset = "g"
include_trailing_delimiter = True
include_folders_as_prefixes = True
soft_deleted = False
versions = True
projection = "full"
Expand Down Expand Up @@ -2048,6 +2049,7 @@ def test_list_blobs_w_explicit_w_user_project(self):
timeout=timeout,
retry=retry,
match_glob=match_glob,
include_folders_as_prefixes=include_folders_as_prefixes,
soft_deleted=soft_deleted,
)

Expand All @@ -2070,6 +2072,7 @@ def test_list_blobs_w_explicit_w_user_project(self):
"versions": versions,
"fields": fields,
"userProject": user_project,
"includeFoldersAsPrefixes": include_folders_as_prefixes,
"softDeleted": soft_deleted,
}
expected_page_start = _blobs_page_start
Expand Down

0 comments on commit 7bb8065

Please sign in to comment.