Skip to content

Commit

Permalink
fix: expect and handle absolute paths as full s3 uri's
Browse files Browse the repository at this point in the history
  • Loading branch information
@cmhulbert
cmhulbert committed Mar 11, 2024
1 parent 45c8f05 commit 5a0ec9b
Showing 1 changed file with 44 additions and 31 deletions.
75 changes: 44 additions & 31 deletions src/main/java/org/janelia/saalfeldlab/n5/s3/AmazonS3KeyValueAccess.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -47,6 +47,7 @@
import java.util.List;
import java.util.stream.Collectors;

import com.amazonaws.services.s3.AmazonS3URI;
import org.janelia.saalfeldlab.n5.KeyValueAccess;
import org.janelia.saalfeldlab.n5.LockedChannel;
import org.janelia.saalfeldlab.n5.N5Exception;
Expand Down Expand Up @@ -137,7 +138,17 @@ public AmazonS3KeyValueAccess(final AmazonS3 s3, final URI containerURI, final b
@Override
public String[] components(final String path) {

final String[] baseComponents = path.split("/");
/* If the path is a valid URI with a scheme then use it to get the key. Otherwise,
* use the path directly, assuming it's a path only */
String key = path;
try {
final URI uri = URI.create(path);
final String scheme = uri.getScheme();
if (scheme != null && !scheme.isEmpty())
key = AmazonS3Utils.getS3Key(uri);
} catch (Throwable ignore) {}

final String[] baseComponents = removeLeadingSlash(key).split("/");
if (baseComponents.length <= 1)
return baseComponents;
return Arrays.stream(baseComponents)
Expand Down Expand Up @@ -169,10 +180,11 @@ public String compose(final String... components) {
@Override
public String compose(final URI uri, final String... components) {

final String[] uriComponents = new String[components.length + 1];
System.arraycopy(components, 0, uriComponents, 1, components.length);
uriComponents[0] = AmazonS3Utils.getS3Key(uri);
return compose(uriComponents);
try {
return uriResolve(uri, compose(components)).toString();
} catch (URISyntaxException x) {
throw new IllegalArgumentException(x.getMessage(), x);
}
}

@Override
Expand All @@ -195,10 +207,7 @@ public String relativize(final String path, final String base) {
final URI baseAsUri = uri("/" + base);
final URI pathAsUri = uri("/" + path);
final URI relativeUri = baseAsUri.relativize(pathAsUri);
final String normalizedString = normalize(relativeUri.toString());
final URI asUri = uri(normalizedString);
final String s3Key = AmazonS3Utils.getS3Key(asUri);
return s3Key;
return relativeUri.getPath();
} catch (final URISyntaxException e) {
throw new N5Exception("Cannot relativize path (" + path + ") with base (" + base + ")", e);
}
Expand Down Expand Up @@ -229,11 +238,16 @@ public String normalize(final String path) {
@Override
public URI uri(final String normalPath) throws URISyntaxException {

return uriResolve(containerURI, normalPath);
}

private URI uriResolve(URI uri, String normalPath) throws URISyntaxException {

if (normalize(normalPath).equals(normalize("/")))
return containerURI;
return uri;

final Path containerPath = Paths.get(containerURI.getPath());
final Path givenPath = Paths.get(URI.create(normalPath).getPath());
final Path containerPath = Paths.get(uri.getPath());
final Path givenPath = Paths.get(new URI(normalPath).getPath());

final Path resolvedPath = containerPath.resolve(givenPath);
final String[] pathParts = new String[resolvedPath.getNameCount() + 1];
Expand All @@ -243,7 +257,7 @@ public URI uri(final String normalPath) throws URISyntaxException {
}
final String normalResolvedPath = compose(pathParts);

return new URI(containerURI.getScheme(), containerURI.getAuthority(), normalResolvedPath, null, null);
return new URI(uri.getScheme(), uri.getAuthority(), normalResolvedPath, null, null);
}

/**
Expand Down Expand Up @@ -279,14 +293,9 @@ private ListObjectsV2Result queryPrefix(final String prefix) {
*/
private boolean keyExists(final String key) {

/* In very preliminary tests, it appears that the obj listing request with one key max
* returns the correct key, but I'm not confident we can count on that in general.
* HeadObjectFunction (found by Caleb) is probably preferable for that reason. -John
*/
try {
final ObjectMetadata objMeta = new HeadObjectFunction(s3).apply(new GetObjectMetadataRequest(bucketName, key));
return objMeta != null;
} catch (Exception e) {
return s3.doesObjectExist(bucketName, key);
} catch (Throwable e) {
return false;
}
}
Expand Down Expand Up @@ -340,7 +349,8 @@ private static String removeLeadingSlash(final String path) {
@Override
public boolean isDirectory(final String normalPath) {

final String key = removeLeadingSlash(addTrailingSlash(normalPath));
final String s3Key = AmazonS3Utils.getS3Key(normalPath);
final String key = removeLeadingSlash(addTrailingSlash(s3Key));
if (key.equals(normalize("/"))) {
return s3.doesBucketExistV2(bucketName);
}
Expand All @@ -360,19 +370,22 @@ public boolean isDirectory(final String normalPath) {
@Override
public boolean isFile(final String normalPath) {

return !normalPath.endsWith("/") && keyExists(removeLeadingSlash(normalPath));
final String key = AmazonS3Utils.getS3Key(normalPath);
return !key.endsWith("/") && keyExists(removeLeadingSlash(key));
}

@Override
public LockedChannel lockForReading(final String normalPath) {

return new S3ObjectChannel(removeLeadingSlash(normalPath), true);
final String key = AmazonS3Utils.getS3Key(normalPath);
return new S3ObjectChannel(removeLeadingSlash(key), true);
}

@Override
public LockedChannel lockForWriting(final String normalPath) {

return new S3ObjectChannel(removeLeadingSlash(normalPath), false);
final String key = AmazonS3Utils.getS3Key(normalPath);
return new S3ObjectChannel(removeLeadingSlash(key), false);
}

@Override
Expand All @@ -387,8 +400,9 @@ private String[] list(final String normalPath, final boolean onlyDirectories) {
throw new N5Exception.N5IOException(normalPath + " is not a valid group");
}

final String pathKey = AmazonS3Utils.getS3Key(normalPath);
final List<String> subGroups = new ArrayList<>();
final String prefix = removeLeadingSlash(addTrailingSlash(normalPath));
final String prefix = removeLeadingSlash(addTrailingSlash(pathKey));
final ListObjectsV2Request listObjectsRequest = new ListObjectsV2Request()
.withBucketName(bucketName)
.withPrefix(prefix)
Expand Down Expand Up @@ -440,7 +454,7 @@ public void delete(final String normalPath) {
return;

// remove bucket when deleting "/"
if (normalPath.equals(normalize("/"))) {
if (AmazonS3Utils.getS3Key(normalPath).equals(normalize("/"))) {

// need to delete all objects before deleting the bucket
// see: https://docs.aws.amazon.com/AmazonS3/latest/userguide/delete-bucket.html
Expand All @@ -466,14 +480,13 @@ public void delete(final String normalPath) {
return;
}

final String path = removeLeadingSlash(normalPath);

if (!path.endsWith("/")) {
final String key = removeLeadingSlash(AmazonS3Utils.getS3Key(normalPath));
if (!key.endsWith("/")) {
s3.deleteObjects(new DeleteObjectsRequest(bucketName)
.withKeys(path));
.withKeys(key));
}

final String prefix = addTrailingSlash(path);
final String prefix = addTrailingSlash(key);
final ListObjectsV2Request listObjectsRequest = new ListObjectsV2Request()
.withBucketName(bucketName)
.withPrefix(prefix);
Expand Down

0 comments on commit 5a0ec9b

Please sign in to comment.