From f84541d62c837f63fd0c30cf2cf9bd38d5b827fd Mon Sep 17 00:00:00 2001 From: Caleb Hulbert Date: Tue, 10 Dec 2024 10:23:46 -0500 Subject: [PATCH 1/3] don't list if where not necessarry --- .../n5/s3/AmazonS3KeyValueAccess.java | 42 ++++++++++++++++++- 1 file changed, 41 insertions(+), 1 deletion(-) diff --git a/src/main/java/org/janelia/saalfeldlab/n5/s3/AmazonS3KeyValueAccess.java b/src/main/java/org/janelia/saalfeldlab/n5/s3/AmazonS3KeyValueAccess.java index b51f184..dcaa2ff 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/s3/AmazonS3KeyValueAccess.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/s3/AmazonS3KeyValueAccess.java @@ -73,6 +73,9 @@ public class AmazonS3KeyValueAccess implements KeyValueAccess { private final URI containerURI; private final String bucketName; + private final boolean createBucket; + private Boolean bucketCheckedAndExists = null; + private static URI uncheckedContainterLocationStringToURI(String uri) { try { @@ -119,6 +122,7 @@ public AmazonS3KeyValueAccess(final AmazonS3 s3, final URI containerURI, final b this.containerURI = containerURI; this.bucketName = AmazonS3Utils.getS3Bucket(containerURI); + this.createBucket = createBucket; if (!s3.doesBucketExistV2(bucketName)) { if (createBucket) { @@ -136,6 +140,38 @@ public AmazonS3KeyValueAccess(final AmazonS3 s3, final URI containerURI, final b } } + private boolean bucketExists() { + if (bucketCheckedAndExists != null) + return bucketCheckedAndExists; + + bucketCheckedAndExists = s3.doesBucketExistV2(bucketName); + return bucketCheckedAndExists; + } + + private void createBucket() { + + if (!createBucket) + throw new N5Exception("Create Bucket Not Allowed"); + + if (bucketExists()) + return; + + + Region region; + try { + region = s3.getRegion(); + } catch (final IllegalStateException e) { + region = Region.US_Standard; + } + try { + s3.createBucket(new CreateBucketRequest(bucketName, region)); + bucketCheckedAndExists = true; + } catch (Exception e) { + throw new N5Exception("Could not create bucket " + bucketName, e); + } + + } + @Override public String[] components(final String path) { @@ -431,6 +467,10 @@ public String[] list(final String normalPath) throws IOException { @Override public void createDirectories(final String normalPath) { + if (!bucketExists() && createBucket){ + createBucket(); + } + String path = ""; for (final String component : components(removeLeadingSlash(normalPath))) { path = addTrailingSlash(compose(path, component)); @@ -607,7 +647,7 @@ public InputStream newInputStream() { try { object = s3.getObject(bucketName, path); } catch (final AmazonServiceException e) { - if (e.getStatusCode() == 404) + if (e.getStatusCode() == 404 || e.getStatusCode() == 403) throw new N5Exception.N5NoSuchKeyException("No such key", e); throw e; } From 5c27419924badab26b484a21803880abbd49a882 Mon Sep 17 00:00:00 2001 From: Caleb Hulbert Date: Thu, 12 Dec 2024 10:37:37 -0500 Subject: [PATCH 2/3] fix: update cached bucket exists flag after delete bucket --- .../n5/s3/AmazonS3KeyValueAccess.java | 25 ++++++++++++++----- 1 file changed, 19 insertions(+), 6 deletions(-) diff --git a/src/main/java/org/janelia/saalfeldlab/n5/s3/AmazonS3KeyValueAccess.java b/src/main/java/org/janelia/saalfeldlab/n5/s3/AmazonS3KeyValueAccess.java index dcaa2ff..a802f80 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/s3/AmazonS3KeyValueAccess.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/s3/AmazonS3KeyValueAccess.java @@ -141,11 +141,10 @@ public AmazonS3KeyValueAccess(final AmazonS3 s3, final URI containerURI, final b } private boolean bucketExists() { - if (bucketCheckedAndExists != null) - return bucketCheckedAndExists; - bucketCheckedAndExists = s3.doesBucketExistV2(bucketName); - return bucketCheckedAndExists; + return bucketCheckedAndExists = bucketCheckedAndExists != null + ? bucketCheckedAndExists + : s3.doesBucketExistV2(bucketName); } private void createBucket() { @@ -156,7 +155,6 @@ private void createBucket() { if (bucketExists()) return; - Region region; try { region = s3.getRegion(); @@ -172,6 +170,21 @@ private void createBucket() { } + private void deleteBucket() { + if (!createBucket) + throw new N5Exception("Delete Bucket Not Allowed"); + + if (!bucketExists()) + return; + + try { + s3.deleteBucket(bucketName); + bucketCheckedAndExists = false; + } catch (Exception e) { + throw new N5Exception("Could not delete bucket " + bucketName, e); + } + } + @Override public String[] components(final String path) { @@ -516,7 +529,7 @@ public void delete(final String normalPath) { } } - s3.deleteBucket(bucketName); + deleteBucket(); return; } From 4b784873db8da372330fbb48f637667bab83e78f Mon Sep 17 00:00:00 2001 From: John Bogovic Date: Thu, 12 Dec 2024 16:23:09 -0500 Subject: [PATCH 3/3] perf: supress AWK SDK warnings see #28 --- .../saalfeldlab/n5/s3/AmazonS3Utils.java | 43 +++++++++++++++++-- 1 file changed, 39 insertions(+), 4 deletions(-) diff --git a/src/main/java/org/janelia/saalfeldlab/n5/s3/AmazonS3Utils.java b/src/main/java/org/janelia/saalfeldlab/n5/s3/AmazonS3Utils.java index 20b75ae..22ab6cc 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/s3/AmazonS3Utils.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/s3/AmazonS3Utils.java @@ -27,8 +27,9 @@ public class AmazonS3Utils { public static final Pattern AWS_ENDPOINT_PATTERN = Pattern.compile("^(.+\\.)?(s3\\..*amazonaws\\.com)", Pattern.CASE_INSENSITIVE); public final static Pattern S3_SCHEME = Pattern.compile("s3", Pattern.CASE_INSENSITIVE); - private AmazonS3Utils() { + private static final String DISABLE_WARNING_KEY = "aws.java.v1.disableDeprecationAnnouncement"; + private AmazonS3Utils() { } public static String getS3Bucket(final String uri) { @@ -102,14 +103,30 @@ private static Regions parseRegion(String stringRegionFromUri) { public static AWSCredentialsProvider getS3Credentials(final AWSCredentials s3Credentials, final boolean s3Anonymous) { + /* + * TODO necessary until we update to AWS SDK (2.x) + * see https://github.com/saalfeldlab/n5-aws-s3/issues/28 + */ + final String initialDisableWarningPropertyValue = System.getProperty(DISABLE_WARNING_KEY); + if( initialDisableWarningPropertyValue == null) + System.setProperty(DISABLE_WARNING_KEY, "true"); + if (s3Credentials != null) { - return new AWSStaticCredentialsProvider(s3Credentials); + final AWSStaticCredentialsProvider provider = new AWSStaticCredentialsProvider(s3Credentials); + resetDisableWarningValue(initialDisableWarningPropertyValue); + return provider; } else { // if not anonymous, try finding credentials - if (!s3Anonymous) - return new DefaultAWSCredentialsProviderChain(); + if (!s3Anonymous) { + final DefaultAWSCredentialsProviderChain provider = new DefaultAWSCredentialsProviderChain(); + resetDisableWarningValue(initialDisableWarningPropertyValue); + return provider; + } else + { + resetDisableWarningValue(initialDisableWarningPropertyValue); return null; + } } } @@ -190,6 +207,15 @@ public static AmazonS3 createS3( @Nullable final Regions region) { final boolean isAmazon = endpointConfiguration == null || AmazonS3Utils.AWS_ENDPOINT_PATTERN.matcher(endpointConfiguration.getServiceEndpoint()).find(); + + /* + * TODO necessary until we update to AWS SDK (2.x) + * see https://github.com/saalfeldlab/n5-aws-s3/issues/28 + */ + final String initialDisableWarningPropertyValue = System.getProperty(DISABLE_WARNING_KEY); + if( initialDisableWarningPropertyValue == null) + System.setProperty(DISABLE_WARNING_KEY, "true"); + final AmazonS3ClientBuilder builder = AmazonS3ClientBuilder.standard(); if (!isAmazon) @@ -220,12 +246,21 @@ else if (region != null) // bucket not detected with anonymous credentials, try detecting credentials // and return it even if it can't detect the bucket, since there's nothing else to do builder.withCredentials(new DefaultAWSCredentialsProviderChain()); + resetDisableWarningValue(initialDisableWarningPropertyValue); return builder.build(); } } + + resetDisableWarningValue(initialDisableWarningPropertyValue); return s3; } + private static void resetDisableWarningValue(final String initialDisableWarningPropertyValue) { + + if (initialDisableWarningPropertyValue == null) + System.clearProperty(DISABLE_WARNING_KEY); + } + private static boolean canListBucket(final AmazonS3 s3, final String bucket) { final ListObjectsV2Request request = new ListObjectsV2Request();