diff --git a/src/main/java/net/iponweb/disthene/reader/DistheneReader.java b/src/main/java/net/iponweb/disthene/reader/DistheneReader.java index 58b2fc60..7d8131b1 100644 --- a/src/main/java/net/iponweb/disthene/reader/DistheneReader.java +++ b/src/main/java/net/iponweb/disthene/reader/DistheneReader.java @@ -35,6 +35,7 @@ public class DistheneReader { private static final String PING_PATH = "^/ping\\/?$"; private static final String RENDER_PATH = "^/render\\/?$"; private static final String SEARCH_PATH = "^/search\\/?$"; + private static final String PATHS_STATS_PATH = "^/path_stats\\/?$"; private static Logger logger; @@ -110,6 +111,10 @@ private void run() { SearchHandler searchHandler = new SearchHandler(indexService, statsService); readerServer.registerHandler(SEARCH_PATH, searchHandler); + logger.info("Creating path stats handler"); + PathStatsHandler pathStatsHandler = new PathStatsHandler(indexService, statsService); + readerServer.registerHandler(PATHS_STATS_PATH, pathStatsHandler); + logger.info("Starting reader"); readerServer.run(); diff --git a/src/main/java/net/iponweb/disthene/reader/handler/PathStatsHandler.java b/src/main/java/net/iponweb/disthene/reader/handler/PathStatsHandler.java new file mode 100644 index 00000000..5cddc268 --- /dev/null +++ b/src/main/java/net/iponweb/disthene/reader/handler/PathStatsHandler.java @@ -0,0 +1,109 @@ +package net.iponweb.disthene.reader.handler; + +import com.google.gson.Gson; +import io.netty.buffer.Unpooled; +import io.netty.handler.codec.http.*; +import io.netty.handler.codec.http.multipart.Attribute; +import io.netty.handler.codec.http.multipart.DefaultHttpDataFactory; +import io.netty.handler.codec.http.multipart.HttpPostRequestDecoder; +import net.iponweb.disthene.reader.exceptions.MissingParameterException; +import net.iponweb.disthene.reader.exceptions.ParameterParsingException; +import net.iponweb.disthene.reader.exceptions.TooMuchDataExpectedException; +import net.iponweb.disthene.reader.exceptions.UnsupportedMethodException; +import net.iponweb.disthene.reader.service.index.IndexService; +import net.iponweb.disthene.reader.service.stats.StatsService; +import org.apache.log4j.Logger; + +import java.io.IOException; +import java.nio.charset.Charset; + +/** + * @author Andrei Ivanov + */ +public class PathStatsHandler implements DistheneReaderHandler { + + final static Logger logger = Logger.getLogger(PathStatsHandler.class); + + private IndexService indexService; + private StatsService statsService; + + public PathStatsHandler(IndexService indexService, StatsService statsService) { + this.indexService = indexService; + this.statsService = statsService; + } + + @Override + public FullHttpResponse handle(HttpRequest request) throws ParameterParsingException, TooMuchDataExpectedException { + PathStatsParameters parameters = parse(request); + + statsService.incPathsRequests(parameters.getTenant()); + + String pathsAsJsonArray = indexService.getPathsWithStats(parameters.getTenant(), parameters.getQuery()); + + FullHttpResponse response = new DefaultFullHttpResponse( + HttpVersion.HTTP_1_1, + HttpResponseStatus.OK, + Unpooled.wrappedBuffer(pathsAsJsonArray.getBytes())); + response.headers().set(HttpHeaders.Names.CONTENT_TYPE, "application/json"); + response.headers().set(HttpHeaders.Names.CONTENT_LENGTH, response.content().readableBytes()); + + return response; + } + + private PathStatsParameters parse(HttpRequest request) throws MissingParameterException, UnsupportedMethodException { + if (request.getMethod().equals(HttpMethod.GET)) { + QueryStringDecoder queryStringDecoder = new QueryStringDecoder(request.getUri()); + + PathStatsParameters parameters = new PathStatsParameters(); + if (queryStringDecoder.parameters().get("tenant") != null) { + parameters.setTenant(queryStringDecoder.parameters().get("tenant").get(0)); + } else { + // assume tenant "NONE" + parameters.setTenant("NONE"); + logger.debug("No tenant in request. Assuming value of NONE"); + } + if (queryStringDecoder.parameters().get("query") != null) { + parameters.setQuery(queryStringDecoder.parameters().get("query").get(0)); + } else { + throw new MissingParameterException("Query parameter is missing"); + } + + return parameters; + } else if (request.getMethod().equals(HttpMethod.POST)) { + PathStatsParameters parameters = new PathStatsParameters(); + ((HttpContent) request).content().resetReaderIndex(); + HttpPostRequestDecoder decoder = new HttpPostRequestDecoder(new DefaultHttpDataFactory(false), request); + + try { + parameters.setTenant(((Attribute) decoder.getBodyHttpData("tenant")).getValue()); + parameters.setQuery(((Attribute) decoder.getBodyHttpData("query")).getValue()); + } catch (IOException e) { + throw new MissingParameterException("Some of the parameters are missing: " + request.getMethod().name()); + } + return parameters; + } else { + throw new UnsupportedMethodException("Method is not supported: " + request.getMethod().name()); + } + } + + private class PathStatsParameters { + private String tenant; + private String query; + + public String getTenant() { + return tenant; + } + + public void setTenant(String tenant) { + this.tenant = tenant; + } + + public String getQuery() { + return query; + } + + public void setQuery(String query) { + this.query = query; + } + } +} diff --git a/src/main/java/net/iponweb/disthene/reader/service/index/IndexService.java b/src/main/java/net/iponweb/disthene/reader/service/index/IndexService.java index e64776e3..19ae10f9 100644 --- a/src/main/java/net/iponweb/disthene/reader/service/index/IndexService.java +++ b/src/main/java/net/iponweb/disthene/reader/service/index/IndexService.java @@ -5,6 +5,7 @@ import net.iponweb.disthene.reader.exceptions.TooMuchDataExpectedException; import net.iponweb.disthene.reader.utils.WildcardUtil; import org.apache.log4j.Logger; +import org.elasticsearch.action.count.CountResponse; import org.elasticsearch.action.search.SearchResponse; import org.elasticsearch.client.transport.TransportClient; import org.elasticsearch.common.settings.ImmutableSettings; @@ -134,6 +135,55 @@ public String getSearchPathsAsString(String tenant, String regEx, int limit) { return Joiner.on(",").skipNulls().join(paths); } + public String getPathsWithStats(String tenant, String wildcard) throws TooMuchDataExpectedException { + String regEx = WildcardUtil.getPathsRegExFromWildcard(wildcard); + + SearchResponse response = client.prepareSearch(indexConfiguration.getIndex()) + .setScroll(new TimeValue(indexConfiguration.getTimeout())) + .setSize(indexConfiguration.getScroll()) + .setQuery(QueryBuilders.filteredQuery( + QueryBuilders.regexpQuery("path", regEx), + FilterBuilders.termFilter("tenant", tenant))) + .addField("path") + .execute().actionGet(); + + // if total hits exceeds maximum - abort right away returning empty array + if (response.getHits().totalHits() > indexConfiguration.getMaxPaths()) { + logger.debug("Total number of paths exceeds the limit: " + response.getHits().totalHits()); + throw new TooMuchDataExpectedException("Total number of paths exceeds the limit: " + response.getHits().totalHits() + " (the limit is " + indexConfiguration.getMaxPaths() + ")"); + } + + List paths = new ArrayList<>(); + while (response.getHits().getHits().length > 0) { + for (SearchHit hit : response.getHits()) { + paths.add(String.valueOf(hit.field("path").getValue())); + } + response = client.prepareSearchScroll(response.getScrollId()) + .setScroll(new TimeValue(indexConfiguration.getTimeout())) + .execute().actionGet(); + } + + Collections.sort(paths); + + // we got the paths. Now let's get the counts + List result = new ArrayList<>(); + for (String path : paths) { + CountResponse countResponse = client.prepareCount(indexConfiguration.getIndex()) + .setQuery(QueryBuilders.filteredQuery( + QueryBuilders.regexpQuery("path", path + "\\..*"), + FilterBuilders.boolFilter() + .must(FilterBuilders.termFilter("tenant", tenant)) + .must(FilterBuilders.termFilter("leaf", true)))) + .execute().actionGet(); + long count = countResponse.getCount(); + result.add("{\"path\": \"" + path + "\",\"count\":" + countResponse.getCount() + "}"); + } + + + return "[" + joiner.join(result) + "]"; + } + + public void shutdown() { client.close(); }