Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

Apply filters to S3Controller getSubdirs #1171

Merged
merged 2 commits into from
Apr 26, 2023
Merged

Apply filters to S3Controller getSubdirs #1171

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
8dbc3e5
Add tests for filtering of s3 files
tdrwenski Apr 26, 2023
5cc767a
Use filter for s3 sub directories in ControllerS3
tdrwenski Apr 26, 2023
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion cdm/s3/src/main/java/thredds/filesystem/s3/ControllerS3.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -121,7 +121,7 @@ public Iterator<MFile> getSubdirs(CollectionConfig mc, boolean recheck) {
logger.error("Error creating MFile for {} bucket {}", commonPrefix, initialUri.getBucket(), e);
}
}
return mFiles.iterator();
return new FilteredIterator(mc, mFiles.iterator(), true);
}

@Override
Expand Down
55 changes: 52 additions & 3 deletions cdm/s3/src/test/java/thredds/filesystem/s3/TestControllerS3.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -18,6 +18,8 @@
import software.amazon.awssdk.regions.Region;
import thredds.inventory.CollectionConfig;
import thredds.inventory.MFile;
import thredds.inventory.MFileFilter;
import thredds.inventory.filter.WildcardMatchOnName;
import ucar.unidata.io.s3.CdmS3Uri;
import ucar.unidata.io.s3.S3TestsCommon;
import ucar.unidata.io.s3.TestS3Read;
Expand Down Expand Up @@ -275,6 +277,41 @@ public void testGetSubdirsWithoutDelimiterOsdc() throws URISyntaxException {
checkSubdirsCount(uri, 0);
}

@Test
public void shouldFilterTopFiles() throws URISyntaxException {
final CdmS3Uri uri = new CdmS3Uri(S3TestsCommon.THREDDS_TEST_BUCKET + "?test-dataset-scan/" + DELIMITER_FRAGMENT);

final CollectionConfig noFilter = new CollectionConfig(uri.getBucket(), uri.toString(), true, null, null);
assertThat(topInventoryCount(noFilter)).isEqualTo(3);

final MFileFilter filter = new WildcardMatchOnName("*.nc$");
final CollectionConfig withFilter = new CollectionConfig(uri.getBucket(), uri.toString(), true, filter, null);
assertThat(topInventoryCount(withFilter)).isEqualTo(2);
}

@Test
public void shouldFilterAllFiles() throws URISyntaxException {
final CdmS3Uri uri = new CdmS3Uri(S3TestsCommon.THREDDS_TEST_BUCKET + "?test-dataset-scan/" + DELIMITER_FRAGMENT);

final CollectionConfig noFilter = new CollectionConfig(uri.getBucket(), uri.toString(), true, null, null);
checkInventoryAllCount(noFilter, 8);

final MFileFilter filter = new WildcardMatchOnName("*.nc$");
final CollectionConfig withFilter = new CollectionConfig(uri.getBucket(), uri.toString(), true, filter, null);
checkInventoryAllCount(withFilter, 4);
}

@Test
public void shouldFilterSubDirs() throws URISyntaxException {
final CdmS3Uri uri = new CdmS3Uri(S3TestsCommon.THREDDS_TEST_BUCKET + "?test-dataset-scan/" + DELIMITER_FRAGMENT);
final CollectionConfig noFilter = new CollectionConfig(uri.getBucket(), uri.toString(), true, null, null);
checkSubdirsCount(noFilter, 2);

final MFileFilter filter = new WildcardMatchOnName("sub-dir");
final CollectionConfig withFilter = new CollectionConfig(uri.getBucket(), uri.toString(), true, filter, null);
checkSubdirsCount(withFilter, 1);
}

@AfterClass
public static void teardown() {
System.clearProperty(AWS_REGION_PROP_NAME);
Expand All @@ -298,25 +335,37 @@ private void checkInventoryTopCountAtMost(CdmS3Uri uri, int expectedMaximumCount

private int topInventoryCount(CdmS3Uri uri) {
logger.debug("getInventoryTop: {}", uri);
return topInventoryCount(getCollectionConfig(uri));
}

private int topInventoryCount(CollectionConfig collectionConfig) {
ControllerS3 controller = new ControllerS3();
controller.limit = true;
Iterator<MFile> it = controller.getInventoryTop(getCollectionConfig(uri), false);
Iterator<MFile> it = controller.getInventoryTop(collectionConfig, false);
return countObjects(it);
}

private void checkInventoryAllCount(CdmS3Uri uri, int expectedCount) {
logger.debug("getInventoryAll: {}", uri);
checkInventoryAllCount(getCollectionConfig(uri), expectedCount);
}

private void checkInventoryAllCount(CollectionConfig collectionConfig, int expectedCount) {
ControllerS3 controller = new ControllerS3();
controller.limit = true;
Iterator<MFile> it = controller.getInventoryAll(getCollectionConfig(uri), false);
Iterator<MFile> it = controller.getInventoryAll(collectionConfig, false);
assertThat(countObjects(it)).isEqualTo(expectedCount);
}

private void checkSubdirsCount(CdmS3Uri uri, int expectedCount) {
logger.debug("getSubdirs: {}", uri);
checkSubdirsCount(getCollectionConfig(uri), expectedCount);
}

private void checkSubdirsCount(CollectionConfig collectionConfig, int expectedCount) {
ControllerS3 controller = new ControllerS3();
controller.limit = true;
Iterator<MFile> it = controller.getSubdirs(getCollectionConfig(uri), false);
Iterator<MFile> it = controller.getSubdirs(collectionConfig, false);
assertThat(countObjects(it)).isEqualTo(expectedCount);
}

Expand Down
1 change: 1 addition & 0 deletions cdm/s3/src/test/java/ucar/unidata/io/s3/S3TestsCommon.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -15,4 +15,5 @@ public class S3TestsCommon {
public static final String TOP_LEVEL_GCS_BUCKET = "cdms3://storage.googleapis.com/gcp-public-data-goes-16";
public static final String TOP_LEVEL_OSDC_BUCKET =
"cdms3://griffin-objstore.opensciencedatacloud.org/noaa-goes16-hurricane-archive-2017";
public static final String THREDDS_TEST_BUCKET = "cdms3:thredds-test-data";
}