diff --git a/cdm/s3/src/main/java/thredds/filesystem/s3/ControllerS3.java b/cdm/s3/src/main/java/thredds/filesystem/s3/ControllerS3.java index e422c2c011..c15d03b144 100644 --- a/cdm/s3/src/main/java/thredds/filesystem/s3/ControllerS3.java +++ b/cdm/s3/src/main/java/thredds/filesystem/s3/ControllerS3.java @@ -121,7 +121,7 @@ public Iterator getSubdirs(CollectionConfig mc, boolean recheck) { logger.error("Error creating MFile for {} bucket {}", commonPrefix, initialUri.getBucket(), e); } } - return mFiles.iterator(); + return new FilteredIterator(mc, mFiles.iterator(), true); } @Override diff --git a/cdm/s3/src/test/java/thredds/filesystem/s3/TestControllerS3.java b/cdm/s3/src/test/java/thredds/filesystem/s3/TestControllerS3.java index 8abf7f8742..86a23fa1b4 100644 --- a/cdm/s3/src/test/java/thredds/filesystem/s3/TestControllerS3.java +++ b/cdm/s3/src/test/java/thredds/filesystem/s3/TestControllerS3.java @@ -18,6 +18,8 @@ import software.amazon.awssdk.regions.Region; import thredds.inventory.CollectionConfig; import thredds.inventory.MFile; +import thredds.inventory.MFileFilter; +import thredds.inventory.filter.WildcardMatchOnName; import ucar.unidata.io.s3.CdmS3Uri; import ucar.unidata.io.s3.S3TestsCommon; import ucar.unidata.io.s3.TestS3Read; @@ -275,6 +277,41 @@ public void testGetSubdirsWithoutDelimiterOsdc() throws URISyntaxException { checkSubdirsCount(uri, 0); } + @Test + public void shouldFilterTopFiles() throws URISyntaxException { + final CdmS3Uri uri = new CdmS3Uri(S3TestsCommon.THREDDS_TEST_BUCKET + "?test-dataset-scan/" + DELIMITER_FRAGMENT); + + final CollectionConfig noFilter = new CollectionConfig(uri.getBucket(), uri.toString(), true, null, null); + assertThat(topInventoryCount(noFilter)).isEqualTo(3); + + final MFileFilter filter = new WildcardMatchOnName("*.nc$"); + final CollectionConfig withFilter = new CollectionConfig(uri.getBucket(), uri.toString(), true, filter, null); + assertThat(topInventoryCount(withFilter)).isEqualTo(2); + } + + @Test + public void shouldFilterAllFiles() throws URISyntaxException { + final CdmS3Uri uri = new CdmS3Uri(S3TestsCommon.THREDDS_TEST_BUCKET + "?test-dataset-scan/" + DELIMITER_FRAGMENT); + + final CollectionConfig noFilter = new CollectionConfig(uri.getBucket(), uri.toString(), true, null, null); + checkInventoryAllCount(noFilter, 8); + + final MFileFilter filter = new WildcardMatchOnName("*.nc$"); + final CollectionConfig withFilter = new CollectionConfig(uri.getBucket(), uri.toString(), true, filter, null); + checkInventoryAllCount(withFilter, 4); + } + + @Test + public void shouldFilterSubDirs() throws URISyntaxException { + final CdmS3Uri uri = new CdmS3Uri(S3TestsCommon.THREDDS_TEST_BUCKET + "?test-dataset-scan/" + DELIMITER_FRAGMENT); + final CollectionConfig noFilter = new CollectionConfig(uri.getBucket(), uri.toString(), true, null, null); + checkSubdirsCount(noFilter, 2); + + final MFileFilter filter = new WildcardMatchOnName("sub-dir"); + final CollectionConfig withFilter = new CollectionConfig(uri.getBucket(), uri.toString(), true, filter, null); + checkSubdirsCount(withFilter, 1); + } + @AfterClass public static void teardown() { System.clearProperty(AWS_REGION_PROP_NAME); @@ -298,25 +335,37 @@ private void checkInventoryTopCountAtMost(CdmS3Uri uri, int expectedMaximumCount private int topInventoryCount(CdmS3Uri uri) { logger.debug("getInventoryTop: {}", uri); + return topInventoryCount(getCollectionConfig(uri)); + } + + private int topInventoryCount(CollectionConfig collectionConfig) { ControllerS3 controller = new ControllerS3(); controller.limit = true; - Iterator it = controller.getInventoryTop(getCollectionConfig(uri), false); + Iterator it = controller.getInventoryTop(collectionConfig, false); return countObjects(it); } private void checkInventoryAllCount(CdmS3Uri uri, int expectedCount) { logger.debug("getInventoryAll: {}", uri); + checkInventoryAllCount(getCollectionConfig(uri), expectedCount); + } + + private void checkInventoryAllCount(CollectionConfig collectionConfig, int expectedCount) { ControllerS3 controller = new ControllerS3(); controller.limit = true; - Iterator it = controller.getInventoryAll(getCollectionConfig(uri), false); + Iterator it = controller.getInventoryAll(collectionConfig, false); assertThat(countObjects(it)).isEqualTo(expectedCount); } private void checkSubdirsCount(CdmS3Uri uri, int expectedCount) { logger.debug("getSubdirs: {}", uri); + checkSubdirsCount(getCollectionConfig(uri), expectedCount); + } + + private void checkSubdirsCount(CollectionConfig collectionConfig, int expectedCount) { ControllerS3 controller = new ControllerS3(); controller.limit = true; - Iterator it = controller.getSubdirs(getCollectionConfig(uri), false); + Iterator it = controller.getSubdirs(collectionConfig, false); assertThat(countObjects(it)).isEqualTo(expectedCount); } diff --git a/cdm/s3/src/test/java/ucar/unidata/io/s3/S3TestsCommon.java b/cdm/s3/src/test/java/ucar/unidata/io/s3/S3TestsCommon.java index 8480fc019d..57c2b57939 100644 --- a/cdm/s3/src/test/java/ucar/unidata/io/s3/S3TestsCommon.java +++ b/cdm/s3/src/test/java/ucar/unidata/io/s3/S3TestsCommon.java @@ -15,4 +15,5 @@ public class S3TestsCommon { public static final String TOP_LEVEL_GCS_BUCKET = "cdms3://storage.googleapis.com/gcp-public-data-goes-16"; public static final String TOP_LEVEL_OSDC_BUCKET = "cdms3://griffin-objstore.opensciencedatacloud.org/noaa-goes16-hurricane-archive-2017"; + public static final String THREDDS_TEST_BUCKET = "cdms3:thredds-test-data"; }