#4696 Fix paging of large numbers of data sources

gchq · Jan 28, 2025 · 862c211 · 862c211
1 parent 4ac0895
commit 862c211
Show file tree

Hide file tree

Showing 15 changed files with 208 additions and 275 deletions.
diff --git a/.../stroom-query-common/src/main/java/stroom/query/common/v2/FieldInfoResultPageBuilder.java b/.../stroom-query-common/src/main/java/stroom/query/common/v2/FieldInfoResultPageBuilder.java
@@ -2,7 +2,6 @@
 
 import stroom.datasource.api.v2.FindFieldCriteria;
 import stroom.datasource.api.v2.QueryField;
-import stroom.util.resultpage.InexactResultPageBuilder;
 import stroom.util.resultpage.ResultPageBuilder;
 import stroom.util.shared.ResultPage;
 import stroom.util.string.StringMatcher;
@@ -21,7 +20,7 @@ public class FieldInfoResultPageBuilder {
     private FieldInfoResultPageBuilder(final FindFieldCriteria criteria) {
         stringMatcher = new StringMatcher(criteria.getStringMatch());
         queryable = criteria.getQueryable();
-        resultPageBuilder = new InexactResultPageBuilder<>(criteria.getPageRequest());
+        resultPageBuilder = new ResultPageBuilder<>(criteria.getPageRequest());
     }
 
     public static FieldInfoResultPageBuilder builder(final FindFieldCriteria criteria) {

diff --git a/stroom-query/stroom-query-impl/src/main/java/stroom/query/impl/DataSources.java b/stroom-query/stroom-query-impl/src/main/java/stroom/query/impl/DataSources.java
@@ -30,8 +30,8 @@
 import stroom.util.logging.LambdaLogger;
 import stroom.util.logging.LambdaLoggerFactory;
 import stroom.util.logging.LogUtil;
+import stroom.util.resultpage.ResultPageBuilder;
 import stroom.util.shared.PageRequest;
-import stroom.util.shared.ResultPage.ResultConsumer;
 import stroom.util.string.AceStringMatcher;
 import stroom.util.string.AceStringMatcher.AceMatchResult;
 import stroom.util.string.StringMatcher;
@@ -69,37 +69,40 @@ public class DataSources {
     public void addRows(final PageRequest pageRequest,
                         final String parentPath,
                         final StringMatcher stringMatcher,
-                        final ResultConsumer<QueryHelpRow> resultConsumer) {
+                        final ResultPageBuilder<QueryHelpRow> resultPageBuilder) {
         if (parentPath.isBlank()) {
             final boolean hasChildren = hasChildren(stringMatcher);
             if (hasChildren ||
                 MatchType.ANY.equals(stringMatcher.getMatchType()) ||
                 stringMatcher.match(ROOT.getTitle()).isPresent()) {
-                resultConsumer.add(ROOT.copy().hasChildren(hasChildren).build());
+                resultPageBuilder.add(ROOT.copy().hasChildren(hasChildren).build());
             }
         } else if (parentPath.startsWith(DATA_SOURCE_ID + ".")) {
             final DataSourceProviderRegistry dataSourceProviderRegistry =
                     dataSourceProviderRegistryProvider.get();
-            final ResultPageBuilder<QueryHelpRow> builder =
-                    new ResultPageBuilder<>(pageRequest, Comparator.comparing(QueryHelpRow::getTitle));
+            final TrimmedSortedList<QueryHelpRow> trimmedSortedList =
+                    new TrimmedSortedList<>(pageRequest, Comparator.comparing(QueryHelpRow::getTitle));
             for (final DocRef docRef : dataSourceProviderRegistry.getDataSourceDocRefs()) {
                 if (stringMatcher.match(docRef.getDisplayValue()).isPresent()) {
-                    final QueryHelpRow row = QueryHelpRow
-                            .builder()
-                            .type(QueryHelpType.DATA_SOURCE)
-                            .id(DATA_SOURCE_ID + "." + docRef.getUuid())
-                            .documentType(docRef.getType())
-                            .iconTooltip(docRef.getType() + " - " + docRef.getDisplayValue())
-                            .title(docRef.getDisplayValue())
-                            .data(new QueryHelpDocument(docRef))
-                            .build();
-                    builder.add(row);
+                    for (int i = 0; i < 1000; i++) {
+                        final QueryHelpRow row = QueryHelpRow
+                                .builder()
+                                .type(QueryHelpType.DATA_SOURCE)
+                                .id(DATA_SOURCE_ID + "." + docRef.getUuid() + "dszf")
+                                .documentType(docRef.getType())
+                                .iconTooltip(docRef.getType() + " - " + docRef.getDisplayValue())
+                                .title("asdf " + i + " " + docRef.getDisplayValue())
+                                .data(new QueryHelpDocument(docRef))
+                                .build();
+
+                        trimmedSortedList.add(row);
+                    }
                 }
             }
-            for (final QueryHelpRow row : builder.build().getValues()) {
-                if (!resultConsumer.add(row)) {
-                    break;
-                }
+
+            final List<QueryHelpRow> list = trimmedSortedList.getList();
+            for (final QueryHelpRow row : list) {
+                resultPageBuilder.add(row);
             }
         }
     }

diff --git a/stroom-query/stroom-query-impl/src/main/java/stroom/query/impl/Dictionaries.java b/stroom-query/stroom-query-impl/src/main/java/stroom/query/impl/Dictionaries.java
@@ -31,8 +31,8 @@
 import stroom.util.logging.LambdaLogger;
 import stroom.util.logging.LambdaLoggerFactory;
 import stroom.util.logging.LogUtil;
+import stroom.util.resultpage.ResultPageBuilder;
 import stroom.util.shared.PageRequest;
-import stroom.util.shared.ResultPage.ResultConsumer;
 import stroom.util.string.AceStringMatcher;
 import stroom.util.string.AceStringMatcher.AceMatchResult;
 import stroom.util.string.StringMatcher;
@@ -68,18 +68,18 @@ public Dictionaries(final DictionaryStore dictionaryStore) {
     public void addRows(final PageRequest pageRequest,
                         final String parentPath,
                         final StringMatcher stringMatcher,
-                        final ResultConsumer<QueryHelpRow> resultConsumer) {
+                        final ResultPageBuilder<QueryHelpRow> resultPageBuilder) {
         final List<DocRef> docs = dictionaryStore.list();
         if (parentPath.isBlank()) {
             final boolean hasChildren = hasChildren(docs, stringMatcher);
             if (hasChildren ||
                 MatchType.ANY.equals(stringMatcher.getMatchType()) ||
                 stringMatcher.match(ROOT.getTitle()).isPresent()) {
-                resultConsumer.add(ROOT.copy().hasChildren(hasChildren).build());
+                resultPageBuilder.add(ROOT.copy().hasChildren(hasChildren).build());
             }
         } else if (parentPath.startsWith(DICTIONARY_ID + ".")) {
-            final ResultPageBuilder<QueryHelpRow> builder =
-                    new ResultPageBuilder<>(pageRequest, Comparator.comparing(QueryHelpRow::getTitle));
+            final TrimmedSortedList<QueryHelpRow> trimmedSortedList =
+                    new TrimmedSortedList<>(pageRequest, Comparator.comparing(QueryHelpRow::getTitle));
 
             for (final DocRef docRef : docs) {
                 if (stringMatcher.match(docRef.getDisplayValue()).isPresent()) {
@@ -92,13 +92,13 @@ public void addRows(final PageRequest pageRequest,
                             .title(docRef.getDisplayValue())
                             .data(new QueryHelpDocument(docRef))
                             .build();
-                    builder.add(row);
+                    trimmedSortedList.add(row);
                 }
             }
-            for (final QueryHelpRow row : builder.build().getValues()) {
-                if (!resultConsumer.add(row)) {
-                    break;
-                }
+
+            final List<QueryHelpRow> list = trimmedSortedList.getList();
+            for (final QueryHelpRow row : list) {
+                resultPageBuilder.add(row);
             }
         }
     }

diff --git a/stroom-query/stroom-query-impl/src/main/java/stroom/query/impl/Fields.java b/stroom-query/stroom-query-impl/src/main/java/stroom/query/impl/Fields.java
@@ -34,9 +34,9 @@
 import stroom.util.logging.LambdaLogger;
 import stroom.util.logging.LambdaLoggerFactory;
 import stroom.util.logging.LogUtil;
+import stroom.util.resultpage.ResultPageBuilder;
 import stroom.util.shared.PageRequest;
 import stroom.util.shared.ResultPage;
-import stroom.util.shared.ResultPage.ResultConsumer;
 import stroom.util.string.AceStringMatcher;
 import stroom.util.string.AceStringMatcher.AceMatchResult;
 import stroom.util.string.StringMatcher;
@@ -72,7 +72,7 @@ public class Fields {
     }
 
     public void addRows(final QueryHelpRequest request,
-                        final ResultConsumer<QueryHelpRow> resultConsumer) {
+                        final ResultPageBuilder<QueryHelpRow> resultPageBuilder) {
         final PageRequest pageRequest = request.getPageRequest();
         if (pageRequest.getLength() > 0) {
             final QueryService queryService = queryServiceProvider.get();
@@ -89,14 +89,14 @@ public void addRows(final QueryHelpRequest request,
                             optional.get(),
                             request.getStringMatch(),
                             null);
-                    hasChildren = queryService.findFields(criteria).size() > 0;
+                    hasChildren = !queryService.findFields(criteria).isEmpty();
                 }
 
                 final StringMatcher stringMatcher = new StringMatcher(request.getStringMatch());
                 if (hasChildren ||
                         MatchType.ANY.equals(stringMatcher.getMatchType()) ||
                         stringMatcher.match(ROOT.getTitle()).isPresent()) {
-                    resultConsumer.add(ROOT.copy().hasChildren(hasChildren).build());
+                    resultPageBuilder.add(ROOT.copy().hasChildren(hasChildren).build());
                 }
 
             } else if (request.getParentPath().startsWith(FIELDS_PARENT) && optional.isPresent()) {
@@ -109,7 +109,7 @@ public void addRows(final QueryHelpRequest request,
                         request.getStringMatch(),
                         null);
                 final ResultPage<QueryField> resultPage = queryService.findFields(criteria);
-                resultConsumer.skip(resultPage.getPageStart());
+                resultPageBuilder.skip(resultPage.getPageStart());
                 resultPage.getValues().forEach(fieldInfo -> {
                     final QueryHelpRow row = new QueryHelpRow(
                             QueryHelpType.FIELD,
@@ -119,7 +119,7 @@ public void addRows(final QueryHelpRequest request,
                             null,
                             fieldInfo.getFldName(),
                             new QueryHelpField(fieldInfo));
-                    resultConsumer.add(row);
+                    resultPageBuilder.add(row);
                 });
             }
         }

diff --git a/stroom-query/stroom-query-impl/src/main/java/stroom/query/impl/Functions.java b/stroom-query/stroom-query-impl/src/main/java/stroom/query/impl/Functions.java
@@ -47,9 +47,9 @@
 import stroom.util.logging.LambdaLogger;
 import stroom.util.logging.LambdaLoggerFactory;
 import stroom.util.logging.LogUtil;
+import stroom.util.resultpage.ResultPageBuilder;
 import stroom.util.shared.GwtNullSafe;
 import stroom.util.shared.PageRequest;
-import stroom.util.shared.ResultPage.ResultConsumer;
 import stroom.util.string.AceStringMatcher;
 import stroom.util.string.AceStringMatcher.AceMatchResult;
 import stroom.util.string.StringMatcher;
@@ -189,29 +189,29 @@ private void addSignature(final Map<String, Set<QueryHelpRow>> map,
     public void addRows(final PageRequest pageRequest,
                         final String parentUuid,
                         final StringMatcher stringMatcher,
-                        final ResultConsumer<QueryHelpRow> resultConsumer) {
+                        final ResultPageBuilder<QueryHelpRow> resultPageBuilder) {
         final List<QueryHelpRow> rows = map.getOrDefault(parentUuid, Collections.emptyList());
-        final ResultPageBuilder<QueryHelpRow> builder =
-                new ResultPageBuilder<>(pageRequest, Comparator.comparing(QueryHelpRow::getTitle));
+        final TrimmedSortedList<QueryHelpRow> trimmedSortedList =
+                new TrimmedSortedList<>(pageRequest, Comparator.comparing(QueryHelpRow::getTitle));
         for (final QueryHelpRow row : rows) {
             if (row.isHasChildren()) {
                 if (!hasChildren(row, stringMatcher)) {
                     if (MatchType.ANY.equals(stringMatcher.getMatchType()) ||
-                            match(row, stringMatcher)) {
-                        builder.add(row.copy().hasChildren(false).build());
+                        match(row, stringMatcher)) {
+                        trimmedSortedList.add(row.copy().hasChildren(false).build());
                     }
                 } else {
-                    builder.add(row);
+                    trimmedSortedList.add(row);
                 }
             } else if (MatchType.ANY.equals(stringMatcher.getMatchType()) ||
-                    match(row, stringMatcher)) {
-                builder.add(row);
+                       match(row, stringMatcher)) {
+                trimmedSortedList.add(row);
             }
         }
-        for (final QueryHelpRow row : builder.build().getValues()) {
-            if (!resultConsumer.add(row)) {
-                break;
-            }
+
+        final List<QueryHelpRow> list = trimmedSortedList.getList();
+        for (final QueryHelpRow row : list) {
+            resultPageBuilder.add(row);
         }
     }
 
@@ -284,7 +284,7 @@ private static QueryHelpFunctionSignature convertSignature(
                 }
             }
             final String helpAnchor = functionDef.helpAnchor() == null
-                    || FunctionDef.UNDEFINED.equals(functionDef.helpAnchor())
+                                      || FunctionDef.UNDEFINED.equals(functionDef.helpAnchor())
                     ? null
                     : functionDef.helpAnchor();
 
@@ -614,7 +614,7 @@ public String buildInfoHtml(final QueryHelpFunctionSignature signature) {
             final UiConfig uiConfig = uiConfigProvider.get();
             if (uiConfig.getHelpUrl() != null && uiConfig.getHelpSubPathStroomQueryLanguage() != null) {
                 addHelpLinkToInfo(signature, uiConfig.getHelpUrl() +
-                        uiConfig.getHelpSubPathStroomQueryLanguage(), detail);
+                                             uiConfig.getHelpSubPathStroomQueryLanguage(), detail);
             }
         }
         return detail.build();
@@ -626,9 +626,9 @@ private static void addHelpLinkToInfo(final QueryHelpFunctionSignature signature
         htmlBuilder.append("For more information see the ");
         htmlBuilder.appendLink(
                 helpUrlBase +
-                        signature.getPrimaryCategory().toLowerCase().replace(" ", "-") +
-                        "#" +
-                        functionSignatureToAnchor(signature),
+                signature.getPrimaryCategory().toLowerCase().replace(" ", "-") +
+                "#" +
+                functionSignatureToAnchor(signature),
                 "Help Documentation");
         htmlBuilder.append(".");
     }
@@ -678,7 +678,7 @@ private static boolean isSymbolicFunction(final String name) {
             return !Character.isAlphabetic(name.charAt(0));
         } else if (name.length() <= 2) {
             return !Character.isAlphabetic(name.charAt(0))
-                    && !Character.isAlphabetic(name.charAt(1));
+                   && !Character.isAlphabetic(name.charAt(1));
         } else {
             return false;
         }

diff --git a/stroom-query/stroom-query-impl/src/main/java/stroom/query/impl/QueryResourceImpl.java b/stroom-query/stroom-query-impl/src/main/java/stroom/query/impl/QueryResourceImpl.java
@@ -34,7 +34,6 @@
 import stroom.query.shared.QuerySearchRequest;
 import stroom.util.logging.LambdaLogger;
 import stroom.util.logging.LambdaLoggerFactory;
-import stroom.util.resultpage.InexactResultPageBuilder;
 import stroom.util.resultpage.ResultPageBuilder;
 import stroom.util.shared.EntityServiceException;
 import stroom.util.shared.PageRequest;
@@ -186,7 +185,7 @@ public ResultPage<QueryHelpRow> fetchQueryHelpItems(final QueryHelpRequest reque
         final String parentPath = request.getParentPath();
         final StringMatcher stringMatcher = new StringMatcher(request.getStringMatch());
         final ResultPageBuilder<QueryHelpRow> resultPageBuilder =
-                new InexactResultPageBuilder<>(request.getPageRequest());
+                new ResultPageBuilder<>(request.getPageRequest());
         PageRequest pageRequest = request.getPageRequest();
         if (request.isTypeIncluded(QueryHelpType.DATA_SOURCE)) {
             dataSourcesProvider.get().addRows(pageRequest, parentPath, stringMatcher, resultPageBuilder);

diff --git a/stroom-query/stroom-query-impl/src/main/java/stroom/query/impl/ResultPageBuilder.java b/stroom-query/stroom-query-impl/src/main/java/stroom/query/impl/ResultPageBuilder.java