From d1ab755d225a4f51f7d73da88736a92b3b61e907 Mon Sep 17 00:00:00 2001 From: Caleb Hulbert Date: Mon, 24 Jun 2024 10:01:11 -0400 Subject: [PATCH] pref: reduce the number of requests to the server `exist` is cheaper if you can stop after an `isFile` request, so do that first `list` call deduplicates `listObjectsV2` call that was performed via `isDirectory` --- .../saalfeldlab/n5/s3/AmazonS3KeyValueAccess.java | 13 ++++++------- 1 file changed, 6 insertions(+), 7 deletions(-) diff --git a/src/main/java/org/janelia/saalfeldlab/n5/s3/AmazonS3KeyValueAccess.java b/src/main/java/org/janelia/saalfeldlab/n5/s3/AmazonS3KeyValueAccess.java index abfbcbf..2e276c9 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/s3/AmazonS3KeyValueAccess.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/s3/AmazonS3KeyValueAccess.java @@ -273,7 +273,7 @@ private URI uriResolve(URI uri, String normalPath) throws URISyntaxException { @Override public boolean exists(final String normalPath) { - return isDirectory(normalPath) || isFile(normalPath); + return isFile(normalPath) || isDirectory(normalPath); } private ListObjectsV2Result queryPrefix(final String prefix) { @@ -396,10 +396,6 @@ public String[] listDirectories(final String normalPath) { private String[] list(final String normalPath, final boolean onlyDirectories) { - if (!isDirectory(normalPath)) { - throw new N5Exception.N5IOException(normalPath + " is not a valid group"); - } - final String pathKey = AmazonS3Utils.getS3Key(normalPath); final List subGroups = new ArrayList<>(); final String prefix = removeLeadingSlash(addTrailingSlash(pathKey)); @@ -407,9 +403,10 @@ private String[] list(final String normalPath, final boolean onlyDirectories) { .withBucketName(bucketName) .withPrefix(prefix) .withDelimiter("/"); - ListObjectsV2Result objectsListing; + ListObjectsV2Result objectsListing = s3.listObjectsV2(listObjectsRequest); + if (objectsListing.getKeyCount() <= 0) + throw new N5Exception.N5IOException(normalPath + " is not a valid group"); do { - objectsListing = s3.listObjectsV2(listObjectsRequest); for (final String commonPrefix : objectsListing.getCommonPrefixes()) { if (!onlyDirectories || commonPrefix.endsWith("/")) { final String relativePath = relativize(commonPrefix, prefix); @@ -418,6 +415,8 @@ private String[] list(final String normalPath, final boolean onlyDirectories) { } } listObjectsRequest.setContinuationToken(objectsListing.getNextContinuationToken()); + if (objectsListing.isTruncated()) + objectsListing = s3.listObjectsV2(listObjectsRequest); } while (objectsListing.isTruncated()); return subGroups.toArray(new String[subGroups.size()]); }