Merge branch 'maint-5.x' into scale-offset

Unidata · May 9, 2023 · c4c7ea1 · c4c7ea1
2 parents 4f033f0 + 413a6f0
commit c4c7ea1
Show file tree

Hide file tree

Showing 5 changed files with 38 additions and 9 deletions.
diff --git a/cdm/core/src/main/java/ucar/nc2/NetcdfFiles.java b/cdm/core/src/main/java/ucar/nc2/NetcdfFiles.java
@@ -400,7 +400,7 @@ private static ucar.unidata.io.RandomAccessFile downloadAndDecompress(ucar.unida
    * @return RandomAccessFile for the object at location
    */
   public static ucar.unidata.io.RandomAccessFile getRaf(String location, int buffer_size) throws IOException {
-    String uriString = location.trim();
+    String uriString = removeFragment(location.trim());
     if (buffer_size <= 0)
       buffer_size = default_buffersize;
 
@@ -473,6 +473,10 @@ public static ucar.unidata.io.RandomAccessFile getRaf(String location, int buffe
     return raf;
   }
 
+  private static String removeFragment(String uriString) {
+    return uriString.split("#")[0];
+  }
+
   private static boolean looksCompressed(String filename) {
     // return true if filename ends with a compressed suffix or contains a compressed suffix followed by a delimiter
     // (i.e. path to a compressed entry)

diff --git a/cdm/s3/src/main/java/thredds/filesystem/s3/ControllerS3.java b/cdm/s3/src/main/java/thredds/filesystem/s3/ControllerS3.java
@@ -141,24 +141,27 @@ private static class FilteredIterator implements Iterator<MFile> {
       this.mc = mc;
       this.wantDirs = wantDirs;
       this.fullInventory = false;
+      this.next = nextFilteredFile();
     }
 
     FilteredIterator(CollectionConfig mc, Iterator<MFile> iter, boolean wantDirs, boolean fullInventory) {
       this.orgIter = iter;
       this.mc = mc;
       this.wantDirs = wantDirs;
       this.fullInventory = fullInventory;
+      this.next = nextFilteredFile();
     }
 
     public boolean hasNext() {
-      next = nextFilteredFile();
       return (next != null);
     }
 
     public MFile next() {
       if (next == null)
         throw new NoSuchElementException();
-      return next;
+      final MFile mFile = next;
+      next = nextFilteredFile();
+      return mFile;
     }
 
     public void remove() {

diff --git a/cdm/s3/src/main/java/thredds/inventory/s3/MFileS3.java b/cdm/s3/src/main/java/thredds/inventory/s3/MFileS3.java
@@ -354,9 +354,9 @@ public ResponseInputStream<GetObjectResponse> getInputStream() {
 
   @Override
   public void writeToStream(OutputStream outputStream) throws IOException {
-    ResponseInputStream<GetObjectResponse> responseInputStream = getInputStream();
-
-    IO.copy(responseInputStream, outputStream);
+    try (ResponseInputStream<GetObjectResponse> responseInputStream = getInputStream()) {
+      IO.copy(responseInputStream, outputStream);
+    }
   }
 
   @Override

diff --git a/cdm/s3/src/test/java/thredds/filesystem/s3/TestControllerS3.java b/cdm/s3/src/test/java/thredds/filesystem/s3/TestControllerS3.java
@@ -64,6 +64,21 @@ public static void setup() {
 
   }
 
+  @Test
+  public void shouldReturnSameValueFromHasNext() throws URISyntaxException {
+    final CdmS3Uri uri = new CdmS3Uri("cdms3:thredds-test-data");
+    final MFileFilter filter = new WildcardMatchOnName("testData.nc");
+    final CollectionConfig collectionConfig = new CollectionConfig(uri.getBucket(), uri.toString(), true, filter, null);
+    final ControllerS3 controller = new ControllerS3();
+    final Iterator<MFile> iterator = controller.getInventoryTop(collectionConfig, false);
+
+    assertThat(iterator.hasNext()).isTrue();
+    assertThat(iterator.hasNext()).isTrue();
+    iterator.next();
+    assertThat(iterator.hasNext()).isFalse();
+    assertThat(iterator.hasNext()).isFalse();
+  }
+
   //////////////////////
   // getInventoryTop() tests
   //

diff --git a/cdm/s3/src/test/java/ucar/unidata/io/s3/TestS3ExternalCompressionRead.java b/cdm/s3/src/test/java/ucar/unidata/io/s3/TestS3ExternalCompressionRead.java
@@ -19,13 +19,20 @@
 
 @Category(NeedsExternalResource.class)
 public class TestS3ExternalCompressionRead {
+  private static final String compressedObject = "cdms3:noaa-nexrad-level2?1991/07/20/KTLX/KTLX19910720_160529.gz";
+  private static final String fragment = "#delimiter=/";
 
   @Test
   public void testCompressedObjectRead() throws IOException {
-    String bucket = "noaa-nexrad-level2";
-    String key = "1991/07/20/KTLX/KTLX19910720_160529.gz";
-    String s3uri = "cdms3:" + bucket + "?" + key;
+    testCompressedObjectRead(compressedObject);
+  }
+
+  @Test
+  public void testCompressedObjectReadWithDelimiterFragment() throws IOException {
+    testCompressedObjectRead(compressedObject + fragment);
+  }
 
+  private static void testCompressedObjectRead(String s3uri) throws IOException {
     System.setProperty(S3TestsCommon.AWS_REGION_PROP_NAME, S3TestsCommon.AWS_G16_REGION);
     try (NetcdfFile ncfile = NetcdfFiles.open(s3uri)) {