Skip to content

Commit

Permalink
update frontend for multiple subgraphs
Browse files Browse the repository at this point in the history
  • Loading branch information
@jamesamcl
jamesamcl committed Aug 18, 2024
1 parent e99928e commit 2e3f59b
Show file tree
Hide file tree
Showing 19 changed files with 353 additions and 95 deletions.
47 changes: 34 additions & 13 deletions grebi_api/src/main/java/uk/ac/ebi/grebi/GrebiApi.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -14,7 +14,10 @@
import org.springframework.data.domain.PageRequest;
import uk.ac.ebi.grebi.repo.GrebiNeoRepo;
import uk.ac.ebi.grebi.db.GrebiSolrQuery;
import uk.ac.ebi.grebi.db.ResolverClient;
import uk.ac.ebi.grebi.db.SummaryClient;
import uk.ac.ebi.grebi.repo.GrebiSolrRepo;
import uk.ac.ebi.grebi.repo.GrebiSummaryRepo;


public class GrebiApi {
Expand All @@ -23,12 +26,22 @@ public static void main(String[] args) throws ParseException, org.apache.commons

final GrebiNeoRepo neo = new GrebiNeoRepo();
final GrebiSolrRepo solr = new GrebiSolrRepo();
final GrebiSummaryRepo summary = new GrebiSummaryRepo();

Gson gson = new Gson();

var edgeTypes = neo.getEdgeTypes();
var stats = neo.getStats();

var rocksDbSubgraphs = (new ResolverClient()).getSubgraphs();
var solrSubgraphs = solr.getSubgraphs();
var summarySubgraphs = summary.getSubgraphs();

if(new HashSet<>(List.of(rocksDbSubgraphs, solrSubgraphs, summarySubgraphs)).size() != 1) {
throw new RuntimeException("RocksDB/Solr/the summary jsons do not seem to contain the same subgraphs. Found: " + String.join(",", rocksDbSubgraphs) + " for RocksDB (from resolver service) and " + String.join(",", solrSubgraphs) + " for Solr (from list of solr cores) and " + String.join(",", summarySubgraphs) + " for the summary jsons (from summary server)");
}

System.out.println("Found subgraphs: " + String.join(",", solrSubgraphs));

Javalin.create(config -> {
config.bundledPlugins.enableCors(cors -> {
cors.addRule(CorsPluginConfig.CorsRule::anyHost);
Expand All @@ -42,31 +55,39 @@ public static void main(String[] args) throws ParseException, org.apache.commons
ctx.contentType("application/json");
ctx.result(gson.toJson(stats));
})
.get("/api/v1/nodes/{nodeId}", ctx -> {
.get("/api/v1/subgraphs", ctx -> {
ctx.contentType("application/json");
ctx.result(gson.toJson(solrSubgraphs));
})
.get("/api/v1/subgraphs/{subgraph}", ctx -> {
ctx.contentType("application/json");
ctx.result(gson.toJson(summary.getSummary(ctx.pathParam("subgraph"))));
})
.get("/api/v1/subgraphs/{subgraph}/nodes/{nodeId}", ctx -> {
ctx.contentType("application/json");
ctx.result("{}");

var q = new GrebiSolrQuery();
q.addFilter("grebi:nodeId", List.of(ctx.pathParam("nodeId")), SearchType.WHOLE_FIELD, false);

var res = solr.getFirstNode(q);
var res = solr.getFirstNode(ctx.pathParam("subgraph"), q);

ctx.contentType("application/json");
ctx.result(gson.toJson(res));
})
.get("/api/v1/nodes/{nodeId}/incoming_edges", ctx -> {
.get("/api/v1/subgraphs/{subgraph}/nodes/{nodeId}/incoming_edges", ctx -> {
ctx.contentType("application/json");
ctx.result(gson.toJson(neo.getIncomingEdges(ctx.pathParam("nodeId"))));
})
.get("/api/v1/edge_types", ctx -> {
ctx.contentType("application/json");
ctx.result(gson.toJson(edgeTypes));
ctx.result(gson.toJson(neo.getIncomingEdges(ctx.pathParam("subgraph"), ctx.pathParam("nodeId"))));
})
// .get("/api/v1/edge_types", ctx -> {
// ctx.contentType("application/json");
// ctx.result(gson.toJson(edgeTypes));
// })
.get("/api/v1/collections", ctx -> {
ctx.contentType("application/json");
ctx.result("{}");
})
.get("/api/v1/search", ctx -> {
.get("/api/v1/subgraphs/{subgraph}/search", ctx -> {
var q = new GrebiSolrQuery();
q.setSearchText(ctx.queryParam("q"));
q.setExactMatch(false);
Expand Down Expand Up @@ -105,12 +126,12 @@ public static void main(String[] args) throws ParseException, org.apache.commons
size = "10";
}
var page = PageRequest.of(Integer.parseInt(page_num), Integer.parseInt(size));
var res = solr.searchNodesPaginated(q, page);
var res = solr.searchNodesPaginated(ctx.pathParam("subgraph"), q, page);
ctx.contentType("application/json");
ctx.result(gson.toJson(res));
})
.get("/api/v1/suggest", ctx -> {
var res = solr.autocomplete(ctx.queryParam("q"));
.get("/api/v1/subgraphs/{subgraph}/suggest", ctx -> {
var res = solr.autocomplete(ctx.pathParam("subgraph"), ctx.queryParam("q"));
ctx.contentType("application/json");
ctx.result(gson.toJson(res));
})
Expand Down
43 changes: 33 additions & 10 deletions grebi_api/src/main/java/uk/ac/ebi/grebi/db/GrebiSolrClient.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -6,9 +6,12 @@
import org.apache.solr.client.solrj.SolrQuery;
import org.apache.solr.client.solrj.SolrServerException;
import org.apache.solr.client.solrj.impl.HttpSolrClient;
import org.apache.solr.client.solrj.request.CoreAdminRequest;
import org.apache.solr.client.solrj.response.CoreAdminResponse;
import org.apache.solr.client.solrj.response.FacetField;
import org.apache.solr.client.solrj.response.QueryResponse;
import org.apache.solr.common.SolrDocument;
import org.apache.solr.common.params.CoreAdminParams.CoreAdminAction;
import org.apache.solr.common.params.SolrParams;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
Expand All @@ -20,6 +23,8 @@
import java.util.LinkedHashMap;
import java.util.List;
import java.util.Map;
import java.util.Set;
import java.util.HashSet;
import java.util.stream.Collectors;

public class GrebiSolrClient {
Expand All @@ -36,9 +41,27 @@ public static String getSolrHost() {
return "http://localhost:8983/";
}

public GrebiFacetedResultsPage<SolrDocument> searchSolrPaginated(GrebiSolrQuery query, Pageable pageable) {
public Set<String> listCores() {

QueryResponse qr = runSolrQuery(query, pageable);
CoreAdminRequest request = new CoreAdminRequest();
request.setAction(CoreAdminAction.STATUS);
try {
org.apache.solr.client.solrj.SolrClient mySolrClient = new HttpSolrClient.Builder(getSolrHost() + "/solr/").build();
var cores = request.process(mySolrClient);
Set<String> ret = new HashSet<String>();
for (int i = 0; i < cores.getCoreStatus().size(); i++) {
ret.add(cores.getCoreStatus().getName(i));
}
return ret;
} catch (Exception e) {
e.printStackTrace();
}
return null;
}

public GrebiFacetedResultsPage<SolrDocument> searchSolrPaginated(String coreName, GrebiSolrQuery query, Pageable pageable) {

QueryResponse qr = runSolrQuery(coreName, query, pageable);

Map<String, Map<String, Long>> facetFieldToCounts = new LinkedHashMap<>();

Expand All @@ -64,9 +87,9 @@ public GrebiFacetedResultsPage<SolrDocument> searchSolrPaginated(GrebiSolrQuery
qr.getResults().getNumFound());
}

public SolrDocument getFirst(GrebiSolrQuery query) {
public SolrDocument getFirst(String coreName, GrebiSolrQuery query) {

QueryResponse qr = runSolrQuery(query, null);
QueryResponse qr = runSolrQuery(coreName, query, null);

if (qr.getResults().getNumFound() < 1) {
logger.info("Expected at least 1 result for solr getFirst for solr query = {}", query.constructQuery().jsonStr());
Expand All @@ -76,11 +99,11 @@ public SolrDocument getFirst(GrebiSolrQuery query) {
return qr.getResults().get(0);
}

public QueryResponse runSolrQuery(GrebiSolrQuery query, Pageable pageable) {
return runSolrQuery(query.constructQuery(), pageable);
public QueryResponse runSolrQuery(String coreName, GrebiSolrQuery query, Pageable pageable) {
return runSolrQuery(coreName, query.constructQuery(), pageable);
}

public QueryResponse runSolrQuery(SolrQuery query, Pageable pageable) {
public QueryResponse runSolrQuery(String coreName, SolrQuery query, Pageable pageable) {

if (pageable != null) {
query.setStart((int) pageable.getOffset());
Expand All @@ -92,7 +115,7 @@ public QueryResponse runSolrQuery(SolrQuery query, Pageable pageable) {
logger.info("solr query urldecoded: {}", URLDecoder.decode(query.toQueryString()));
logger.info("solr host: {}", SOLR_HOST);

org.apache.solr.client.solrj.SolrClient mySolrClient = new HttpSolrClient.Builder(getSolrHost() + "/solr/grebi_nodes").build();
org.apache.solr.client.solrj.SolrClient mySolrClient = new HttpSolrClient.Builder(getSolrHost() + "/solr/" + coreName).build();

QueryResponse qr = null;
try {
Expand All @@ -112,8 +135,8 @@ public QueryResponse runSolrQuery(SolrQuery query, Pageable pageable) {
return qr;
}

public List<String> autocomplete(String q) {
org.apache.solr.client.solrj.SolrClient mySolrClient = new HttpSolrClient.Builder(getSolrHost() + "/solr/grebi_autocomplete").build();
public List<String> autocomplete(String subgraph, String q) {
org.apache.solr.client.solrj.SolrClient mySolrClient = new HttpSolrClient.Builder(getSolrHost() + "/solr/grebi_autocomplete_" + subgraph).build();

SolrQuery query = new SolrQuery();
query.set("defType", "edismax");
Expand Down
27 changes: 23 additions & 4 deletions grebi_api/src/main/java/uk/ac/ebi/grebi/db/ResolverClient.java
View file
Original file line number Diff line number Diff line change
@@ -1,9 +1,11 @@

package uk.ac.ebi.grebi.db;

import java.io.IOException;
import java.util.Collection;
import java.util.List;
import java.util.Map;
import java.util.Set;

import com.google.gson.JsonElement;
import java.util.List;
Expand All @@ -15,8 +17,10 @@
import com.google.gson.internal.LinkedTreeMap;
import org.apache.http.HttpEntity;
import org.apache.http.HttpResponse;
import org.apache.http.client.ClientProtocolException;
import org.apache.http.client.HttpClient;
import org.apache.http.client.methods.HttpPost;
import org.apache.http.client.methods.HttpGet;
import org.apache.http.impl.client.HttpClientBuilder;
import org.apache.http.util.EntityUtils;
import org.apache.http.entity.StringEntity;
Expand All @@ -34,7 +38,22 @@ public static String getResolverHost() {
return "http://localhost:8080/";
}

public Map<String, Map<String,JsonElement>> resolveToMap(Collection<String> ids) {
public Set<String> getSubgraphs() {
HttpClient client = HttpClientBuilder.create().build();
HttpGet request = new HttpGet(getResolverHost() + "/subgraphs");
HttpResponse response;
try {
response = client.execute(request);
HttpEntity entity = response.getEntity();
String json = EntityUtils.toString(entity);
return new Gson().fromJson(json, Set.class);
} catch (Exception e) {
e.printStackTrace();
}
return null;
}

public Map<String, Map<String,JsonElement>> resolveToMap(String subgraph, Collection<String> ids) {

Stopwatch timer = Stopwatch.createStarted();

Expand All @@ -43,7 +62,7 @@ public Map<String, Map<String,JsonElement>> resolveToMap(Collection<String> ids)

String resolverHost = getResolverHost();

HttpPost request = new HttpPost(resolverHost + "/resolve");
HttpPost request = new HttpPost(resolverHost + "/" + subgraph + "/resolve");
request.setEntity(new StringEntity(gson.toJson(ids), ContentType.APPLICATION_JSON));

// System.out.println("calling resolver at " + resolverHost + "/resolve" + " with " + gson.toJson(ids));
Expand All @@ -69,9 +88,9 @@ public Map<String, Map<String,JsonElement>> resolveToMap(Collection<String> ids)
return null;
}

public List<Map<String, JsonElement>> resolveToList(Collection<String> ids) {
public List<Map<String, JsonElement>> resolveToList(String subgraph, Collection<String> ids) {

var resolved = resolveToMap(ids);
var resolved = resolveToMap(subgraph, ids);

return ids.stream().map(id -> resolved.get(id)).collect(Collectors.toList());
}
Expand Down
58 changes: 58 additions & 0 deletions grebi_api/src/main/java/uk/ac/ebi/grebi/db/SummaryClient.java
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,58 @@

package uk.ac.ebi.grebi.db;

import java.io.IOException;
import java.util.Collection;
import java.util.List;
import java.util.Map;
import java.util.Set;

import com.google.gson.JsonElement;
import java.util.List;
import java.util.Map;
import java.util.stream.Collectors;

import com.google.gson.JsonElement;
import com.google.gson.Gson;
import com.google.gson.internal.LinkedTreeMap;
import org.apache.http.HttpEntity;
import org.apache.http.HttpResponse;
import org.apache.http.client.ClientProtocolException;
import org.apache.http.client.HttpClient;
import org.apache.http.client.methods.HttpPost;
import org.apache.http.client.methods.HttpGet;
import org.apache.http.impl.client.HttpClientBuilder;
import org.apache.http.util.EntityUtils;
import org.apache.http.entity.StringEntity;
import org.apache.http.entity.ContentType;
import com.google.common.base.Stopwatch;

public class SummaryClient {

static final String SUMMARY_HOST = System.getenv("GREBI_SUMMARY_HOST");


public static String getSummaryHost() {
if (SUMMARY_HOST != null)
return SUMMARY_HOST;
return "http://localhost:8081/";
}

public Map<String,JsonElement> getSummaries() {
HttpClient client = HttpClientBuilder.create().build();
HttpGet request = new HttpGet(getSummaryHost());
HttpResponse response;
try {
response = client.execute(request);
HttpEntity entity = response.getEntity();
String json = EntityUtils.toString(entity);
return new Gson().fromJson(json, JsonElement.class).getAsJsonObject().asMap();
} catch (Exception e) {
e.printStackTrace();
}
return null;
}



}
17 changes: 2 additions & 15 deletions grebi_api/src/main/java/uk/ac/ebi/grebi/repo/GrebiNeoRepo.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -27,22 +27,8 @@ public class GrebiNeoRepo {
public GrebiNeoRepo() throws IOException {}

final String STATS_QUERY = new String(GrebiApi.class.getResourceAsStream("/cypher/stats.cypher").readAllBytes(), StandardCharsets.UTF_8);
final String SEARCH_QUERY = new String(GrebiApi.class.getResourceAsStream("/cypher/search.cypher").readAllBytes(), StandardCharsets.UTF_8);
final String PROPS_QUERY = new String(GrebiApi.class.getResourceAsStream("/cypher/props.cypher").readAllBytes(), StandardCharsets.UTF_8);
final String INCOMING_EDGES_QUERY = new String(GrebiApi.class.getResourceAsStream("/cypher/incoming_edges.cypher").readAllBytes(), StandardCharsets.UTF_8);

public Map<String,JsonElement> getEdgeTypes() {
EagerResult props_res = neo4jClient.getDriver().executableQuery(PROPS_QUERY).withConfig(QueryConfig.builder().withDatabase("neo4j").build()).execute();
Map<String,JsonElement> edgeTypes = new TreeMap<>();
for(var r : props_res.records().get(0).values()) {
//JsonObject prop_def = gson.fromJson(r.asString(), JsonElement.class).getAsJsonObject();
//edgeTypes.put(prop_def.get("grebi:nodeId").getAsString(), prop_def);
//Map<String,Object> o = (Map<String,Object>)r.get("n").asObject();
//edgeTypes.put((String)o.get("grebi:nodeId"), gson.toJsonTree(o));
}
return edgeTypes;
}

public Map<String,Object> getStats() {
EagerResult props_res = neo4jClient.getDriver().executableQuery(STATS_QUERY).withConfig(QueryConfig.builder().withDatabase("neo4j").build()).execute();
return props_res.records().get(0).values().get(0).asMap();
Expand All @@ -60,12 +46,13 @@ public EdgeAndNode(Map<String,JsonElement> edge, Map<String,JsonElement> node) {
}
}

public List<EdgeAndNode> getIncomingEdges(String nodeId) {
public List<EdgeAndNode> getIncomingEdges(String subgraph, String nodeId) {
EagerResult res = neo4jClient.getDriver().executableQuery(INCOMING_EDGES_QUERY)
.withParameters(Map.of("nodeId", nodeId))
.withConfig(QueryConfig.builder().withDatabase("neo4j").build()).execute();

var resolved = resolver.resolveToMap(
subgraph,
res.records().stream().flatMap(record -> {
var props = record.asMap();
return List.of((String) props.get("otherId"), (String) props.get("edgeId")).stream();
Expand Down
Loading

0 comments on commit 2e3f59b

Please sign in to comment.