optimize ES performance & bump up version

EinsamHauer · Jan 21, 2021 · 84ad595 · 84ad595
1 parent 8444717
commit 84ad595
Show file tree

Hide file tree

Showing 2 changed files with 11 additions and 31 deletions.
diff --git a/pom.xml b/pom.xml
@@ -4,7 +4,7 @@
     <groupId>net.iponweb.disthene.reader</groupId>
     <artifactId>disthene-reader</artifactId>
     <packaging>jar</packaging>
-    <version>1.0.17</version>
+    <version>1.0.18</version>
     <name>disthene-reader</name>
     <url>http://maven.apache.org</url>
     <dependencies>
@@ -51,7 +51,7 @@
         <dependency>
             <groupId>org.elasticsearch</groupId>
             <artifactId>elasticsearch</artifactId>
-            <version>1.3.4</version>
+            <version>1.7.3</version>
         </dependency>
         <dependency>
             <groupId>net.jpountz.lz4</groupId>

diff --git a/src/main/java/net/iponweb/disthene/reader/service/index/IndexService.java b/src/main/java/net/iponweb/disthene/reader/service/index/IndexService.java
@@ -11,7 +11,6 @@
 import org.elasticsearch.common.settings.ImmutableSettings;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.transport.InetSocketTransportAddress;
-import org.elasticsearch.common.unit.TimeValue;
 import org.elasticsearch.index.query.FilterBuilders;
 import org.elasticsearch.index.query.QueryBuilders;
 import org.elasticsearch.search.SearchHit;
@@ -61,8 +60,7 @@ public List<String> getPaths(String tenant, List<String> wildcards) throws TooMu
             String regEx = Joiner.on("|").skipNulls().join(regExs);
 
             SearchResponse response = client.prepareSearch(indexConfiguration.getIndex())
-                    .setScroll(new TimeValue(indexConfiguration.getTimeout()))
-                    .setSize(indexConfiguration.getScroll())
+                    .setSize(indexConfiguration.getMaxPaths())
                     .setQuery(QueryBuilders.filteredQuery(QueryBuilders.regexpQuery("path", regEx),
                             FilterBuilders.termFilter("tenant", tenant)))
                     .addField("path")
@@ -74,13 +72,8 @@ public List<String> getPaths(String tenant, List<String> wildcards) throws TooMu
                 throw new TooMuchDataExpectedException("Total number of paths exceeds the limit: " + response.getHits().totalHits() + " (the limit is " + indexConfiguration.getMaxPaths() + ")");
             }
 
-            while (response.getHits().getHits().length > 0) {
-                for (SearchHit hit : response.getHits()) {
-                    result.add(hit.field("path").getValue());
-                }
-                response = client.prepareSearchScroll(response.getScrollId())
-                        .setScroll(new TimeValue(indexConfiguration.getTimeout()))
-                        .execute().actionGet();
+            for (SearchHit hit : response.getHits()) {
+                result.add(hit.field("path").getValue());
             }
         }
 
@@ -91,8 +84,7 @@ public String getPathsAsJsonArray(String tenant, String wildcard) throws TooMuch
         String regEx = WildcardUtil.getPathsRegExFromWildcard(wildcard);
 
         SearchResponse response = client.prepareSearch(indexConfiguration.getIndex())
-                .setScroll(new TimeValue(indexConfiguration.getTimeout()))
-                .setSize(indexConfiguration.getScroll())
+                .setSize(indexConfiguration.getMaxPaths())
                 .setQuery(QueryBuilders.filteredQuery(
                         QueryBuilders.regexpQuery("path", regEx),
                         FilterBuilders.termFilter("tenant", tenant)))
@@ -105,21 +97,15 @@ public String getPathsAsJsonArray(String tenant, String wildcard) throws TooMuch
         }
 
         List<String> paths = new ArrayList<>();
-        while (response.getHits().getHits().length > 0) {
-            for (SearchHit hit : response.getHits()) {
-                paths.add(hit.getSourceAsString());
-            }
-            response = client.prepareSearchScroll(response.getScrollId())
-                    .setScroll(new TimeValue(indexConfiguration.getTimeout()))
-                    .execute().actionGet();
+        for (SearchHit hit : response.getHits()) {
+            paths.add(hit.getSourceAsString());
         }
 
         return "[" + joiner.join(paths) + "]";
     }
 
     public String getSearchPathsAsString(String tenant, String regEx, int limit) {
         SearchResponse response = client.prepareSearch(indexConfiguration.getIndex())
-                .setScroll(new TimeValue(indexConfiguration.getTimeout()))
                 .setSize(limit)
                 .setQuery(QueryBuilders.filteredQuery(
                         QueryBuilders.regexpQuery("path", regEx),
@@ -139,8 +125,7 @@ public String getPathsWithStats(String tenant, String wildcard) throws TooMuchDa
         String regEx = WildcardUtil.getPathsRegExFromWildcard(wildcard);
 
         SearchResponse response = client.prepareSearch(indexConfiguration.getIndex())
-                .setScroll(new TimeValue(indexConfiguration.getTimeout()))
-                .setSize(indexConfiguration.getScroll())
+                .setSize(indexConfiguration.getMaxPaths())
                 .setQuery(QueryBuilders.filteredQuery(
                         QueryBuilders.regexpQuery("path", regEx),
                         FilterBuilders.termFilter("tenant", tenant)))
@@ -154,13 +139,8 @@ public String getPathsWithStats(String tenant, String wildcard) throws TooMuchDa
         }
 
         List<String> paths = new ArrayList<>();
-        while (response.getHits().getHits().length > 0) {
-            for (SearchHit hit : response.getHits()) {
-                paths.add(hit.field("path").getValue());
-            }
-            response = client.prepareSearchScroll(response.getScrollId())
-                    .setScroll(new TimeValue(indexConfiguration.getTimeout()))
-                    .execute().actionGet();
+        for (SearchHit hit : response.getHits()) {
+            paths.add(hit.field("path").getValue());
         }
 
         Collections.sort(paths);