Skip to content

Commit

Permalink
pref: reduce the number of requests to the server
Browse files Browse the repository at this point in the history
`exist` is cheaper if you can stop after an `isFile` request, so do that first
 `list` call deduplicates `listObjectsV2` call that was performed via
 `isDirectory`
  • Loading branch information
cmhulbert committed Jun 24, 2024
1 parent 3ce10f1 commit d1ab755
Showing 1 changed file with 6 additions and 7 deletions.
Original file line number Diff line number Diff line change
Expand Up @@ -273,7 +273,7 @@ private URI uriResolve(URI uri, String normalPath) throws URISyntaxException {
@Override
public boolean exists(final String normalPath) {

return isDirectory(normalPath) || isFile(normalPath);
return isFile(normalPath) || isDirectory(normalPath);
}

private ListObjectsV2Result queryPrefix(final String prefix) {
Expand Down Expand Up @@ -396,20 +396,17 @@ public String[] listDirectories(final String normalPath) {

private String[] list(final String normalPath, final boolean onlyDirectories) {

if (!isDirectory(normalPath)) {
throw new N5Exception.N5IOException(normalPath + " is not a valid group");
}

final String pathKey = AmazonS3Utils.getS3Key(normalPath);
final List<String> subGroups = new ArrayList<>();
final String prefix = removeLeadingSlash(addTrailingSlash(pathKey));
final ListObjectsV2Request listObjectsRequest = new ListObjectsV2Request()
.withBucketName(bucketName)
.withPrefix(prefix)
.withDelimiter("/");
ListObjectsV2Result objectsListing;
ListObjectsV2Result objectsListing = s3.listObjectsV2(listObjectsRequest);
if (objectsListing.getKeyCount() <= 0)
throw new N5Exception.N5IOException(normalPath + " is not a valid group");
do {
objectsListing = s3.listObjectsV2(listObjectsRequest);
for (final String commonPrefix : objectsListing.getCommonPrefixes()) {
if (!onlyDirectories || commonPrefix.endsWith("/")) {
final String relativePath = relativize(commonPrefix, prefix);
Expand All @@ -418,6 +415,8 @@ private String[] list(final String normalPath, final boolean onlyDirectories) {
}
}
listObjectsRequest.setContinuationToken(objectsListing.getNextContinuationToken());
if (objectsListing.isTruncated())
objectsListing = s3.listObjectsV2(listObjectsRequest);
} while (objectsListing.isTruncated());
return subGroups.toArray(new String[subGroups.size()]);
}
Expand Down

0 comments on commit d1ab755

Please sign in to comment.