Added basic stats handler

EinsamHauer · Nov 29, 2017 · b48655b · b48655b
1 parent 0dddcad
commit b48655b
Show file tree

Hide file tree

Showing 3 changed files with 164 additions and 0 deletions.
diff --git a/src/main/java/net/iponweb/disthene/reader/DistheneReader.java b/src/main/java/net/iponweb/disthene/reader/DistheneReader.java
@@ -35,6 +35,7 @@ public class DistheneReader {
     private static final String PING_PATH = "^/ping\\/?$";
     private static final String RENDER_PATH = "^/render\\/?$";
     private static final String SEARCH_PATH = "^/search\\/?$";
+    private static final String PATHS_STATS_PATH = "^/path_stats\\/?$";
 
     private static Logger logger;
 
@@ -110,6 +111,10 @@ private void run() {
             SearchHandler searchHandler = new SearchHandler(indexService, statsService);
             readerServer.registerHandler(SEARCH_PATH, searchHandler);
 
+            logger.info("Creating path stats handler");
+            PathStatsHandler pathStatsHandler = new PathStatsHandler(indexService, statsService);
+            readerServer.registerHandler(PATHS_STATS_PATH, pathStatsHandler);
+
             logger.info("Starting reader");
             readerServer.run();
 

diff --git a/src/main/java/net/iponweb/disthene/reader/handler/PathStatsHandler.java b/src/main/java/net/iponweb/disthene/reader/handler/PathStatsHandler.java
@@ -0,0 +1,109 @@
+package net.iponweb.disthene.reader.handler;
+
+import com.google.gson.Gson;
+import io.netty.buffer.Unpooled;
+import io.netty.handler.codec.http.*;
+import io.netty.handler.codec.http.multipart.Attribute;
+import io.netty.handler.codec.http.multipart.DefaultHttpDataFactory;
+import io.netty.handler.codec.http.multipart.HttpPostRequestDecoder;
+import net.iponweb.disthene.reader.exceptions.MissingParameterException;
+import net.iponweb.disthene.reader.exceptions.ParameterParsingException;
+import net.iponweb.disthene.reader.exceptions.TooMuchDataExpectedException;
+import net.iponweb.disthene.reader.exceptions.UnsupportedMethodException;
+import net.iponweb.disthene.reader.service.index.IndexService;
+import net.iponweb.disthene.reader.service.stats.StatsService;
+import org.apache.log4j.Logger;
+
+import java.io.IOException;
+import java.nio.charset.Charset;
+
+/**
+ * @author Andrei Ivanov
+ */
+public class PathStatsHandler implements DistheneReaderHandler {
+
+    final static Logger logger = Logger.getLogger(PathStatsHandler.class);
+
+    private IndexService indexService;
+    private StatsService statsService;
+
+    public PathStatsHandler(IndexService indexService, StatsService statsService) {
+        this.indexService = indexService;
+        this.statsService = statsService;
+    }
+
+    @Override
+    public FullHttpResponse handle(HttpRequest request) throws ParameterParsingException, TooMuchDataExpectedException {
+        PathStatsParameters parameters = parse(request);
+
+        statsService.incPathsRequests(parameters.getTenant());
+
+        String pathsAsJsonArray = indexService.getPathsWithStats(parameters.getTenant(), parameters.getQuery());
+
+        FullHttpResponse response = new DefaultFullHttpResponse(
+                HttpVersion.HTTP_1_1,
+                HttpResponseStatus.OK,
+                Unpooled.wrappedBuffer(pathsAsJsonArray.getBytes()));
+        response.headers().set(HttpHeaders.Names.CONTENT_TYPE, "application/json");
+        response.headers().set(HttpHeaders.Names.CONTENT_LENGTH, response.content().readableBytes());
+
+        return response;
+    }
+
+    private PathStatsParameters parse(HttpRequest request) throws MissingParameterException, UnsupportedMethodException {
+        if (request.getMethod().equals(HttpMethod.GET)) {
+            QueryStringDecoder queryStringDecoder = new QueryStringDecoder(request.getUri());
+
+            PathStatsParameters parameters = new PathStatsParameters();
+            if (queryStringDecoder.parameters().get("tenant") != null) {
+                parameters.setTenant(queryStringDecoder.parameters().get("tenant").get(0));
+            } else {
+                // assume tenant "NONE"
+                parameters.setTenant("NONE");
+                logger.debug("No tenant in request. Assuming value of NONE");
+            }
+            if (queryStringDecoder.parameters().get("query") != null) {
+                parameters.setQuery(queryStringDecoder.parameters().get("query").get(0));
+            } else {
+                throw new MissingParameterException("Query parameter is missing");
+            }
+
+            return parameters;
+        } else if (request.getMethod().equals(HttpMethod.POST)) {
+            PathStatsParameters parameters = new PathStatsParameters();
+            ((HttpContent) request).content().resetReaderIndex();
+            HttpPostRequestDecoder decoder = new HttpPostRequestDecoder(new DefaultHttpDataFactory(false), request);
+
+            try {
+                parameters.setTenant(((Attribute) decoder.getBodyHttpData("tenant")).getValue());
+                parameters.setQuery(((Attribute) decoder.getBodyHttpData("query")).getValue());
+            } catch (IOException e) {
+                throw new MissingParameterException("Some of the parameters are missing: " + request.getMethod().name());
+            }
+            return parameters;
+        } else {
+            throw new UnsupportedMethodException("Method is not supported: " + request.getMethod().name());
+        }
+    }
+
+    private class PathStatsParameters {
+        private String tenant;
+        private String query;
+
+        public String getTenant() {
+            return tenant;
+        }
+
+        public void setTenant(String tenant) {
+            this.tenant = tenant;
+        }
+
+        public String getQuery() {
+            return query;
+        }
+
+        public void setQuery(String query) {
+            this.query = query;
+        }
+    }
+}
diff --git a/src/main/java/net/iponweb/disthene/reader/service/index/IndexService.java b/src/main/java/net/iponweb/disthene/reader/service/index/IndexService.java
@@ -5,6 +5,7 @@
 import net.iponweb.disthene.reader.exceptions.TooMuchDataExpectedException;
 import net.iponweb.disthene.reader.utils.WildcardUtil;
 import org.apache.log4j.Logger;
+import org.elasticsearch.action.count.CountResponse;
 import org.elasticsearch.action.search.SearchResponse;
 import org.elasticsearch.client.transport.TransportClient;
 import org.elasticsearch.common.settings.ImmutableSettings;
@@ -134,6 +135,55 @@ public String getSearchPathsAsString(String tenant, String regEx, int limit) {
         return Joiner.on(",").skipNulls().join(paths);
     }
 
+    public String getPathsWithStats(String tenant, String wildcard) throws TooMuchDataExpectedException {
+        String regEx = WildcardUtil.getPathsRegExFromWildcard(wildcard);
+
+        SearchResponse response = client.prepareSearch(indexConfiguration.getIndex())
+                .setScroll(new TimeValue(indexConfiguration.getTimeout()))
+                .setSize(indexConfiguration.getScroll())
+                .setQuery(QueryBuilders.filteredQuery(
+                        QueryBuilders.regexpQuery("path", regEx),
+                        FilterBuilders.termFilter("tenant", tenant)))
+                .addField("path")
+                .execute().actionGet();
+
+        // if total hits exceeds maximum - abort right away returning empty array
+        if (response.getHits().totalHits() > indexConfiguration.getMaxPaths()) {
+            logger.debug("Total number of paths exceeds the limit: " + response.getHits().totalHits());
+            throw new TooMuchDataExpectedException("Total number of paths exceeds the limit: " + response.getHits().totalHits() + " (the limit is " + indexConfiguration.getMaxPaths() + ")");
+        }
+
+        List<String> paths = new ArrayList<>();
+        while (response.getHits().getHits().length > 0) {
+            for (SearchHit hit : response.getHits()) {
+                paths.add(String.valueOf(hit.field("path").getValue()));
+            }
+            response = client.prepareSearchScroll(response.getScrollId())
+                    .setScroll(new TimeValue(indexConfiguration.getTimeout()))
+                    .execute().actionGet();
+        }
+
+        Collections.sort(paths);
+
+        // we got the paths. Now let's get the counts
+        List<String> result = new ArrayList<>();
+        for (String path : paths) {
+            CountResponse countResponse = client.prepareCount(indexConfiguration.getIndex())
+                    .setQuery(QueryBuilders.filteredQuery(
+                            QueryBuilders.regexpQuery("path", path + "\\..*"),
+                            FilterBuilders.boolFilter()
+                                .must(FilterBuilders.termFilter("tenant", tenant))
+                                .must(FilterBuilders.termFilter("leaf", true))))
+                    .execute().actionGet();
+            long count = countResponse.getCount();
+            result.add("{\"path\": \"" + path + "\",\"count\":" + countResponse.getCount() + "}");
+        }
+
+
+        return "[" + joiner.join(result) + "]";
+    }
+
+
     public void shutdown() {
         client.close();
     }