From 2e3f59b6e1f3c02954388cdc75fc6ca8b3b65efd Mon Sep 17 00:00:00 2001 From: James McLaughlin Date: Sun, 18 Aug 2024 23:31:41 +0100 Subject: [PATCH] update frontend for multiple subgraphs --- .../main/java/uk/ac/ebi/grebi/GrebiApi.java | 47 ++++++++++----- .../uk/ac/ebi/grebi/db/GrebiSolrClient.java | 43 ++++++++++---- .../uk/ac/ebi/grebi/db/ResolverClient.java | 27 +++++++-- .../uk/ac/ebi/grebi/db/SummaryClient.java | 58 +++++++++++++++++++ .../uk/ac/ebi/grebi/repo/GrebiNeoRepo.java | 17 +----- .../uk/ac/ebi/grebi/repo/GrebiSolrRepo.java | 48 +++++++++------ .../ac/ebi/grebi/repo/GrebiSummaryRepo.java | 30 ++++++++++ .../src/main/resources/cypher/props.cypher | 5 -- .../GrebiResolverSvc.java | 42 ++++++++++---- grebi_summary_service/Dockerfile | 10 ++++ .../GrebiSummarySvc.java | 41 +++++++++++++ grebi_ui/src/App.tsx | 5 +- grebi_ui/src/components/ClassExpression.tsx | 4 +- grebi_ui/src/components/SearchBox.tsx | 6 +- grebi_ui/src/model/GraphNode.ts | 6 +- grebi_ui/src/pages/home/Home.tsx | 38 ++++++++++-- grebi_ui/src/pages/node/NodePage.tsx | 6 +- .../src/pages/node/prop_table/PropVals.tsx | 6 +- grebi_ui/src/pages/search/Search.tsx | 9 ++- 19 files changed, 353 insertions(+), 95 deletions(-) create mode 100644 grebi_api/src/main/java/uk/ac/ebi/grebi/db/SummaryClient.java create mode 100644 grebi_api/src/main/java/uk/ac/ebi/grebi/repo/GrebiSummaryRepo.java delete mode 100644 grebi_api/src/main/resources/cypher/props.cypher create mode 100644 grebi_summary_service/Dockerfile create mode 100644 grebi_summary_service/src/main/java/uk/ac/ebi/grebi_summary_service/GrebiSummarySvc.java diff --git a/grebi_api/src/main/java/uk/ac/ebi/grebi/GrebiApi.java b/grebi_api/src/main/java/uk/ac/ebi/grebi/GrebiApi.java index b0246c5..829b499 100644 --- a/grebi_api/src/main/java/uk/ac/ebi/grebi/GrebiApi.java +++ b/grebi_api/src/main/java/uk/ac/ebi/grebi/GrebiApi.java @@ -14,7 +14,10 @@ import org.springframework.data.domain.PageRequest; import uk.ac.ebi.grebi.repo.GrebiNeoRepo; import uk.ac.ebi.grebi.db.GrebiSolrQuery; +import uk.ac.ebi.grebi.db.ResolverClient; +import uk.ac.ebi.grebi.db.SummaryClient; import uk.ac.ebi.grebi.repo.GrebiSolrRepo; +import uk.ac.ebi.grebi.repo.GrebiSummaryRepo; public class GrebiApi { @@ -23,12 +26,22 @@ public static void main(String[] args) throws ParseException, org.apache.commons final GrebiNeoRepo neo = new GrebiNeoRepo(); final GrebiSolrRepo solr = new GrebiSolrRepo(); + final GrebiSummaryRepo summary = new GrebiSummaryRepo(); Gson gson = new Gson(); - var edgeTypes = neo.getEdgeTypes(); var stats = neo.getStats(); + var rocksDbSubgraphs = (new ResolverClient()).getSubgraphs(); + var solrSubgraphs = solr.getSubgraphs(); + var summarySubgraphs = summary.getSubgraphs(); + + if(new HashSet<>(List.of(rocksDbSubgraphs, solrSubgraphs, summarySubgraphs)).size() != 1) { + throw new RuntimeException("RocksDB/Solr/the summary jsons do not seem to contain the same subgraphs. Found: " + String.join(",", rocksDbSubgraphs) + " for RocksDB (from resolver service) and " + String.join(",", solrSubgraphs) + " for Solr (from list of solr cores) and " + String.join(",", summarySubgraphs) + " for the summary jsons (from summary server)"); + } + + System.out.println("Found subgraphs: " + String.join(",", solrSubgraphs)); + Javalin.create(config -> { config.bundledPlugins.enableCors(cors -> { cors.addRule(CorsPluginConfig.CorsRule::anyHost); @@ -42,31 +55,39 @@ public static void main(String[] args) throws ParseException, org.apache.commons ctx.contentType("application/json"); ctx.result(gson.toJson(stats)); }) - .get("/api/v1/nodes/{nodeId}", ctx -> { + .get("/api/v1/subgraphs", ctx -> { + ctx.contentType("application/json"); + ctx.result(gson.toJson(solrSubgraphs)); + }) + .get("/api/v1/subgraphs/{subgraph}", ctx -> { + ctx.contentType("application/json"); + ctx.result(gson.toJson(summary.getSummary(ctx.pathParam("subgraph")))); + }) + .get("/api/v1/subgraphs/{subgraph}/nodes/{nodeId}", ctx -> { ctx.contentType("application/json"); ctx.result("{}"); var q = new GrebiSolrQuery(); q.addFilter("grebi:nodeId", List.of(ctx.pathParam("nodeId")), SearchType.WHOLE_FIELD, false); - var res = solr.getFirstNode(q); + var res = solr.getFirstNode(ctx.pathParam("subgraph"), q); ctx.contentType("application/json"); ctx.result(gson.toJson(res)); }) - .get("/api/v1/nodes/{nodeId}/incoming_edges", ctx -> { + .get("/api/v1/subgraphs/{subgraph}/nodes/{nodeId}/incoming_edges", ctx -> { ctx.contentType("application/json"); - ctx.result(gson.toJson(neo.getIncomingEdges(ctx.pathParam("nodeId")))); - }) - .get("/api/v1/edge_types", ctx -> { - ctx.contentType("application/json"); - ctx.result(gson.toJson(edgeTypes)); + ctx.result(gson.toJson(neo.getIncomingEdges(ctx.pathParam("subgraph"), ctx.pathParam("nodeId")))); }) +// .get("/api/v1/edge_types", ctx -> { +// ctx.contentType("application/json"); +// ctx.result(gson.toJson(edgeTypes)); +// }) .get("/api/v1/collections", ctx -> { ctx.contentType("application/json"); ctx.result("{}"); }) - .get("/api/v1/search", ctx -> { + .get("/api/v1/subgraphs/{subgraph}/search", ctx -> { var q = new GrebiSolrQuery(); q.setSearchText(ctx.queryParam("q")); q.setExactMatch(false); @@ -105,12 +126,12 @@ public static void main(String[] args) throws ParseException, org.apache.commons size = "10"; } var page = PageRequest.of(Integer.parseInt(page_num), Integer.parseInt(size)); - var res = solr.searchNodesPaginated(q, page); + var res = solr.searchNodesPaginated(ctx.pathParam("subgraph"), q, page); ctx.contentType("application/json"); ctx.result(gson.toJson(res)); }) - .get("/api/v1/suggest", ctx -> { - var res = solr.autocomplete(ctx.queryParam("q")); + .get("/api/v1/subgraphs/{subgraph}/suggest", ctx -> { + var res = solr.autocomplete(ctx.pathParam("subgraph"), ctx.queryParam("q")); ctx.contentType("application/json"); ctx.result(gson.toJson(res)); }) diff --git a/grebi_api/src/main/java/uk/ac/ebi/grebi/db/GrebiSolrClient.java b/grebi_api/src/main/java/uk/ac/ebi/grebi/db/GrebiSolrClient.java index 04c5a28..8bdc967 100644 --- a/grebi_api/src/main/java/uk/ac/ebi/grebi/db/GrebiSolrClient.java +++ b/grebi_api/src/main/java/uk/ac/ebi/grebi/db/GrebiSolrClient.java @@ -6,9 +6,12 @@ import org.apache.solr.client.solrj.SolrQuery; import org.apache.solr.client.solrj.SolrServerException; import org.apache.solr.client.solrj.impl.HttpSolrClient; +import org.apache.solr.client.solrj.request.CoreAdminRequest; +import org.apache.solr.client.solrj.response.CoreAdminResponse; import org.apache.solr.client.solrj.response.FacetField; import org.apache.solr.client.solrj.response.QueryResponse; import org.apache.solr.common.SolrDocument; +import org.apache.solr.common.params.CoreAdminParams.CoreAdminAction; import org.apache.solr.common.params.SolrParams; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -20,6 +23,8 @@ import java.util.LinkedHashMap; import java.util.List; import java.util.Map; +import java.util.Set; +import java.util.HashSet; import java.util.stream.Collectors; public class GrebiSolrClient { @@ -36,9 +41,27 @@ public static String getSolrHost() { return "http://localhost:8983/"; } - public GrebiFacetedResultsPage searchSolrPaginated(GrebiSolrQuery query, Pageable pageable) { + public Set listCores() { - QueryResponse qr = runSolrQuery(query, pageable); + CoreAdminRequest request = new CoreAdminRequest(); + request.setAction(CoreAdminAction.STATUS); + try { + org.apache.solr.client.solrj.SolrClient mySolrClient = new HttpSolrClient.Builder(getSolrHost() + "/solr/").build(); + var cores = request.process(mySolrClient); + Set ret = new HashSet(); + for (int i = 0; i < cores.getCoreStatus().size(); i++) { + ret.add(cores.getCoreStatus().getName(i)); + } + return ret; + } catch (Exception e) { + e.printStackTrace(); + } + return null; + } + + public GrebiFacetedResultsPage searchSolrPaginated(String coreName, GrebiSolrQuery query, Pageable pageable) { + + QueryResponse qr = runSolrQuery(coreName, query, pageable); Map> facetFieldToCounts = new LinkedHashMap<>(); @@ -64,9 +87,9 @@ public GrebiFacetedResultsPage searchSolrPaginated(GrebiSolrQuery qr.getResults().getNumFound()); } - public SolrDocument getFirst(GrebiSolrQuery query) { + public SolrDocument getFirst(String coreName, GrebiSolrQuery query) { - QueryResponse qr = runSolrQuery(query, null); + QueryResponse qr = runSolrQuery(coreName, query, null); if (qr.getResults().getNumFound() < 1) { logger.info("Expected at least 1 result for solr getFirst for solr query = {}", query.constructQuery().jsonStr()); @@ -76,11 +99,11 @@ public SolrDocument getFirst(GrebiSolrQuery query) { return qr.getResults().get(0); } - public QueryResponse runSolrQuery(GrebiSolrQuery query, Pageable pageable) { - return runSolrQuery(query.constructQuery(), pageable); + public QueryResponse runSolrQuery(String coreName, GrebiSolrQuery query, Pageable pageable) { + return runSolrQuery(coreName, query.constructQuery(), pageable); } - public QueryResponse runSolrQuery(SolrQuery query, Pageable pageable) { + public QueryResponse runSolrQuery(String coreName, SolrQuery query, Pageable pageable) { if (pageable != null) { query.setStart((int) pageable.getOffset()); @@ -92,7 +115,7 @@ public QueryResponse runSolrQuery(SolrQuery query, Pageable pageable) { logger.info("solr query urldecoded: {}", URLDecoder.decode(query.toQueryString())); logger.info("solr host: {}", SOLR_HOST); - org.apache.solr.client.solrj.SolrClient mySolrClient = new HttpSolrClient.Builder(getSolrHost() + "/solr/grebi_nodes").build(); + org.apache.solr.client.solrj.SolrClient mySolrClient = new HttpSolrClient.Builder(getSolrHost() + "/solr/" + coreName).build(); QueryResponse qr = null; try { @@ -112,8 +135,8 @@ public QueryResponse runSolrQuery(SolrQuery query, Pageable pageable) { return qr; } - public List autocomplete(String q) { - org.apache.solr.client.solrj.SolrClient mySolrClient = new HttpSolrClient.Builder(getSolrHost() + "/solr/grebi_autocomplete").build(); + public List autocomplete(String subgraph, String q) { + org.apache.solr.client.solrj.SolrClient mySolrClient = new HttpSolrClient.Builder(getSolrHost() + "/solr/grebi_autocomplete_" + subgraph).build(); SolrQuery query = new SolrQuery(); query.set("defType", "edismax"); diff --git a/grebi_api/src/main/java/uk/ac/ebi/grebi/db/ResolverClient.java b/grebi_api/src/main/java/uk/ac/ebi/grebi/db/ResolverClient.java index a046144..57f64ab 100644 --- a/grebi_api/src/main/java/uk/ac/ebi/grebi/db/ResolverClient.java +++ b/grebi_api/src/main/java/uk/ac/ebi/grebi/db/ResolverClient.java @@ -1,9 +1,11 @@ package uk.ac.ebi.grebi.db; +import java.io.IOException; import java.util.Collection; import java.util.List; import java.util.Map; +import java.util.Set; import com.google.gson.JsonElement; import java.util.List; @@ -15,8 +17,10 @@ import com.google.gson.internal.LinkedTreeMap; import org.apache.http.HttpEntity; import org.apache.http.HttpResponse; +import org.apache.http.client.ClientProtocolException; import org.apache.http.client.HttpClient; import org.apache.http.client.methods.HttpPost; +import org.apache.http.client.methods.HttpGet; import org.apache.http.impl.client.HttpClientBuilder; import org.apache.http.util.EntityUtils; import org.apache.http.entity.StringEntity; @@ -34,7 +38,22 @@ public static String getResolverHost() { return "http://localhost:8080/"; } - public Map> resolveToMap(Collection ids) { + public Set getSubgraphs() { + HttpClient client = HttpClientBuilder.create().build(); + HttpGet request = new HttpGet(getResolverHost() + "/subgraphs"); + HttpResponse response; + try { + response = client.execute(request); + HttpEntity entity = response.getEntity(); + String json = EntityUtils.toString(entity); + return new Gson().fromJson(json, Set.class); + } catch (Exception e) { + e.printStackTrace(); + } + return null; + } + + public Map> resolveToMap(String subgraph, Collection ids) { Stopwatch timer = Stopwatch.createStarted(); @@ -43,7 +62,7 @@ public Map> resolveToMap(Collection ids) String resolverHost = getResolverHost(); - HttpPost request = new HttpPost(resolverHost + "/resolve"); + HttpPost request = new HttpPost(resolverHost + "/" + subgraph + "/resolve"); request.setEntity(new StringEntity(gson.toJson(ids), ContentType.APPLICATION_JSON)); // System.out.println("calling resolver at " + resolverHost + "/resolve" + " with " + gson.toJson(ids)); @@ -69,9 +88,9 @@ public Map> resolveToMap(Collection ids) return null; } - public List> resolveToList(Collection ids) { + public List> resolveToList(String subgraph, Collection ids) { - var resolved = resolveToMap(ids); + var resolved = resolveToMap(subgraph, ids); return ids.stream().map(id -> resolved.get(id)).collect(Collectors.toList()); } diff --git a/grebi_api/src/main/java/uk/ac/ebi/grebi/db/SummaryClient.java b/grebi_api/src/main/java/uk/ac/ebi/grebi/db/SummaryClient.java new file mode 100644 index 0000000..7ddc564 --- /dev/null +++ b/grebi_api/src/main/java/uk/ac/ebi/grebi/db/SummaryClient.java @@ -0,0 +1,58 @@ + +package uk.ac.ebi.grebi.db; + +import java.io.IOException; +import java.util.Collection; +import java.util.List; +import java.util.Map; +import java.util.Set; + +import com.google.gson.JsonElement; +import java.util.List; +import java.util.Map; +import java.util.stream.Collectors; + +import com.google.gson.JsonElement; +import com.google.gson.Gson; +import com.google.gson.internal.LinkedTreeMap; +import org.apache.http.HttpEntity; +import org.apache.http.HttpResponse; +import org.apache.http.client.ClientProtocolException; +import org.apache.http.client.HttpClient; +import org.apache.http.client.methods.HttpPost; +import org.apache.http.client.methods.HttpGet; +import org.apache.http.impl.client.HttpClientBuilder; +import org.apache.http.util.EntityUtils; +import org.apache.http.entity.StringEntity; +import org.apache.http.entity.ContentType; +import com.google.common.base.Stopwatch; + +public class SummaryClient { + + static final String SUMMARY_HOST = System.getenv("GREBI_SUMMARY_HOST"); + + + public static String getSummaryHost() { + if (SUMMARY_HOST != null) + return SUMMARY_HOST; + return "http://localhost:8081/"; + } + + public Map getSummaries() { + HttpClient client = HttpClientBuilder.create().build(); + HttpGet request = new HttpGet(getSummaryHost()); + HttpResponse response; + try { + response = client.execute(request); + HttpEntity entity = response.getEntity(); + String json = EntityUtils.toString(entity); + return new Gson().fromJson(json, JsonElement.class).getAsJsonObject().asMap(); + } catch (Exception e) { + e.printStackTrace(); + } + return null; + } + + + +} diff --git a/grebi_api/src/main/java/uk/ac/ebi/grebi/repo/GrebiNeoRepo.java b/grebi_api/src/main/java/uk/ac/ebi/grebi/repo/GrebiNeoRepo.java index 6d5c54b..60db4f7 100644 --- a/grebi_api/src/main/java/uk/ac/ebi/grebi/repo/GrebiNeoRepo.java +++ b/grebi_api/src/main/java/uk/ac/ebi/grebi/repo/GrebiNeoRepo.java @@ -27,22 +27,8 @@ public class GrebiNeoRepo { public GrebiNeoRepo() throws IOException {} final String STATS_QUERY = new String(GrebiApi.class.getResourceAsStream("/cypher/stats.cypher").readAllBytes(), StandardCharsets.UTF_8); - final String SEARCH_QUERY = new String(GrebiApi.class.getResourceAsStream("/cypher/search.cypher").readAllBytes(), StandardCharsets.UTF_8); - final String PROPS_QUERY = new String(GrebiApi.class.getResourceAsStream("/cypher/props.cypher").readAllBytes(), StandardCharsets.UTF_8); final String INCOMING_EDGES_QUERY = new String(GrebiApi.class.getResourceAsStream("/cypher/incoming_edges.cypher").readAllBytes(), StandardCharsets.UTF_8); - public Map getEdgeTypes() { - EagerResult props_res = neo4jClient.getDriver().executableQuery(PROPS_QUERY).withConfig(QueryConfig.builder().withDatabase("neo4j").build()).execute(); - Map edgeTypes = new TreeMap<>(); - for(var r : props_res.records().get(0).values()) { - //JsonObject prop_def = gson.fromJson(r.asString(), JsonElement.class).getAsJsonObject(); - //edgeTypes.put(prop_def.get("grebi:nodeId").getAsString(), prop_def); - //Map o = (Map)r.get("n").asObject(); - //edgeTypes.put((String)o.get("grebi:nodeId"), gson.toJsonTree(o)); - } - return edgeTypes; - } - public Map getStats() { EagerResult props_res = neo4jClient.getDriver().executableQuery(STATS_QUERY).withConfig(QueryConfig.builder().withDatabase("neo4j").build()).execute(); return props_res.records().get(0).values().get(0).asMap(); @@ -60,12 +46,13 @@ public EdgeAndNode(Map edge, Map node) { } } - public List getIncomingEdges(String nodeId) { + public List getIncomingEdges(String subgraph, String nodeId) { EagerResult res = neo4jClient.getDriver().executableQuery(INCOMING_EDGES_QUERY) .withParameters(Map.of("nodeId", nodeId)) .withConfig(QueryConfig.builder().withDatabase("neo4j").build()).execute(); var resolved = resolver.resolveToMap( + subgraph, res.records().stream().flatMap(record -> { var props = record.asMap(); return List.of((String) props.get("otherId"), (String) props.get("edgeId")).stream(); diff --git a/grebi_api/src/main/java/uk/ac/ebi/grebi/repo/GrebiSolrRepo.java b/grebi_api/src/main/java/uk/ac/ebi/grebi/repo/GrebiSolrRepo.java index 323813d..18662a7 100644 --- a/grebi_api/src/main/java/uk/ac/ebi/grebi/repo/GrebiSolrRepo.java +++ b/grebi_api/src/main/java/uk/ac/ebi/grebi/repo/GrebiSolrRepo.java @@ -1,8 +1,7 @@ package uk.ac.ebi.grebi.repo; -import java.util.Collection; -import java.util.List; -import java.util.Map; +import java.util.*; +import java.util.stream.Collector; import java.util.stream.Collectors; import org.apache.solr.common.SolrDocument; @@ -22,45 +21,60 @@ public class GrebiSolrRepo { public GrebiSolrRepo() {} - public List autocomplete(String q) { - return solrClient.autocomplete(q); + public Set getSubgraphs() { + + var cores = solrClient.listCores(); + + var autocompleteCores = cores.stream().filter(core -> core.startsWith("grebi_autocomplete_")).map(core -> core.replace("grebi_autocomplete_", "")).collect(Collectors.toSet()); + var nodesCores = cores.stream().filter(core -> core.startsWith("grebi_nodes_")).map(core -> core.replace("grebi_nodes_", "")).collect(Collectors.toSet()); + var edgesCores = cores.stream().filter(core -> core.startsWith("grebi_edges_")).map(core -> core.replace("grebi_edges_", "")).collect(Collectors.toSet()); + + if(new HashSet<>(List.of(autocompleteCores, nodesCores, edgesCores)).size() != 1) { + throw new RuntimeException("autocomplete, nodes, and edges cores must be present for all subgraphs. Found cores: " + String.join(",", cores)); + } + + return autocompleteCores; // any will do they are identical + } + + public List autocomplete(String subgraph, String q) { + return solrClient.autocomplete(subgraph, q); } - public GrebiFacetedResultsPage> searchNodesPaginated(GrebiSolrQuery query, Pageable pageable) { - return resolveNodeIds(solrClient.searchSolrPaginated(query, pageable)); + public GrebiFacetedResultsPage> searchNodesPaginated(String subgraph, GrebiSolrQuery query, Pageable pageable) { + return resolveNodeIds(subgraph, solrClient.searchSolrPaginated("grebi_nodes_"+subgraph, query, pageable)); } - public Map getFirstNode(GrebiSolrQuery query) { - return resolveNodeId(solrClient.getFirst(query)); + public Map getFirstNode(String subgraph, GrebiSolrQuery query) { + return resolveNodeId(subgraph, solrClient.getFirst("grebi_nodes_"+subgraph, query)); } - private GrebiFacetedResultsPage> resolveNodeIds(GrebiFacetedResultsPage solrDocs) { + private GrebiFacetedResultsPage> resolveNodeIds(String subgraph, GrebiFacetedResultsPage solrDocs) { List ids = solrDocs.map(doc -> doc.getFieldValue("grebi__nodeId").toString()).toList(); - List> vals = resolver.resolveToList(ids); + List> vals = resolver.resolveToList(subgraph, ids); assert(vals.size() == solrDocs.getSize()); return new GrebiFacetedResultsPage<>(vals, solrDocs.facetFieldToCounts, solrDocs.getPageable(), solrDocs.getTotalElements()); } - private Map resolveNodeId(SolrDocument solrDoc) { - return resolver.resolveToList(List.of(solrDoc.getFieldValue("grebi__nodeId").toString())).iterator().next(); + private Map resolveNodeId(String subgraph, SolrDocument solrDoc) { + return resolver.resolveToList(subgraph, List.of(solrDoc.getFieldValue("grebi__nodeId").toString())).iterator().next(); } - private GrebiFacetedResultsPage> resolveEdgeIds(GrebiFacetedResultsPage solrDocs) { + private GrebiFacetedResultsPage> resolveEdgeIds(String subgraph, GrebiFacetedResultsPage solrDocs) { List ids = solrDocs.map(doc -> doc.getFieldValue("grebi__edgeId").toString()).toList(); - List> vals = resolver.resolveToList(ids); + List> vals = resolver.resolveToList(subgraph, ids); assert(vals.size() == solrDocs.getSize()); return new GrebiFacetedResultsPage<>(vals, solrDocs.facetFieldToCounts, solrDocs.getPageable(), solrDocs.getTotalElements()); } - private Map resolveEdgeId(SolrDocument solrDoc) { + private Map resolveEdgeId(String subgraph, SolrDocument solrDoc) { - return resolver.resolveToList(List.of(solrDoc.getFieldValue("grebi__edgeId").toString())).iterator().next(); + return resolver.resolveToList(subgraph, List.of(solrDoc.getFieldValue("grebi__edgeId").toString())).iterator().next(); } } diff --git a/grebi_api/src/main/java/uk/ac/ebi/grebi/repo/GrebiSummaryRepo.java b/grebi_api/src/main/java/uk/ac/ebi/grebi/repo/GrebiSummaryRepo.java new file mode 100644 index 0000000..358ec1b --- /dev/null +++ b/grebi_api/src/main/java/uk/ac/ebi/grebi/repo/GrebiSummaryRepo.java @@ -0,0 +1,30 @@ +package uk.ac.ebi.grebi.repo; + +import java.util.Map; +import java.util.Set; + +import com.google.gson.JsonElement; + +import uk.ac.ebi.grebi.db.SummaryClient; + +public class GrebiSummaryRepo { + + Map subgraph2summary; + + public GrebiSummaryRepo() { + + SummaryClient summaryClient = new SummaryClient(); + subgraph2summary = summaryClient.getSummaries(); + + } + + public Set getSubgraphs() { + return subgraph2summary.keySet(); + } + + public Map getSummary(String subgraph) { + return subgraph2summary.get(subgraph).getAsJsonObject().asMap(); + } + + +} diff --git a/grebi_api/src/main/resources/cypher/props.cypher b/grebi_api/src/main/resources/cypher/props.cypher deleted file mode 100644 index 3387a9c..0000000 --- a/grebi_api/src/main/resources/cypher/props.cypher +++ /dev/null @@ -1,5 +0,0 @@ -CALL db.relationshipTypes() YIELD relationshipType -UNWIND relationshipType AS t -WITH t -MATCH (n:GraphNode { `grebi:nodeId`: t }) -RETURN n diff --git a/grebi_resolver_service/src/main/java/uk/ac/ebi/grebi_resolver_service/GrebiResolverSvc.java b/grebi_resolver_service/src/main/java/uk/ac/ebi/grebi_resolver_service/GrebiResolverSvc.java index 8eb2aee..e90fb32 100644 --- a/grebi_resolver_service/src/main/java/uk/ac/ebi/grebi_resolver_service/GrebiResolverSvc.java +++ b/grebi_resolver_service/src/main/java/uk/ac/ebi/grebi_resolver_service/GrebiResolverSvc.java @@ -10,17 +10,18 @@ import org.rocksdb.RocksDBException; import java.io.InputStreamReader; +import java.io.File; import java.util.ArrayList; +import java.util.Arrays; import java.util.HashMap; import java.util.List; import java.util.Map; public class GrebiResolverSvc { - private static RocksDB rocksDB; + private static Map rocksDBs = new HashMap<>(); public static void main(String[] args) { - Gson gson = new Gson(); RocksDB.loadLibrary(); @@ -28,17 +29,38 @@ public static void main(String[] args) { Options options = new Options(); options.setCreateIfMissing(false); - try { - rocksDB = RocksDB.openReadOnly(options, System.getenv("GREBI_ROCKSDB_PATH")); - } catch (RocksDBException e) { - e.printStackTrace(); - return; + var dirs = Arrays.stream(new File(System.getenv("GREBI_ROCKSDB_SEARCH_PATH")).listFiles()).filter(File::isDirectory).filter(f -> f.getName().endsWith("_rocksdb")).toArray(File[]::new); + + for (File dir : dirs) { + RocksDB rocksDB = null; + try { + rocksDB = RocksDB.openReadOnly(options, dir.getAbsolutePath()); + } catch (RocksDBException e) { + e.printStackTrace(); + return; + } + var subgraph = dir.getName().split("_rocksdb")[0]; + rocksDBs.put(subgraph, rocksDB); + System.out.println("Loaded RocksDB for subgraph " + subgraph + " from " + dir.getAbsolutePath()); } Javalin app = Javalin.create(config -> { }).start(8080); - app.post("/resolve", ctx -> { + app.get("/subgraphs", ctx -> { + ctx.contentType("application/json"); + ctx.result(gson.toJson(rocksDBs.keySet())); + }); + + app.post("/{subgraph}/resolve", ctx -> { + + var subgraph = ctx.pathParam("subgraph"); + var rocksdb = rocksDBs.get(subgraph); + if(rocksdb == null) { + ctx.status(404).result("Subgraph not found"); + return; + } + List paramArray = gson.fromJson(new InputStreamReader(ctx.bodyInputStream()), List.class); List keys = new ArrayList<>(); for (String id : paramArray) { @@ -47,7 +69,7 @@ public static void main(String[] args) { Map results = new HashMap<>(); try { - List values = rocksDB.multiGetAsList(keys); + List values = rocksdb.multiGetAsList(keys); int n = 0; for (byte[] value : values) { byte[] key = keys.get(n++); @@ -67,7 +89,7 @@ public static void main(String[] args) { }); Runtime.getRuntime().addShutdownHook(new Thread(() -> { - if (rocksDB != null) { + for (RocksDB rocksDB : rocksDBs.values()) { rocksDB.close(); } })); diff --git a/grebi_summary_service/Dockerfile b/grebi_summary_service/Dockerfile new file mode 100644 index 0000000..5464eee --- /dev/null +++ b/grebi_summary_service/Dockerfile @@ -0,0 +1,10 @@ + +FROM maven:3.9.6-amazoncorretto-17 + +COPY . /opt/ +RUN cd /opt/ && ls && mvn clean package assembly:single -DskipTests + +EXPOSE 8080 +ENTRYPOINT ["java", "-jar", "/opt/target/grebi_summary_service-1.0-SNAPSHOT-jar-with-dependencies.jar"] + + diff --git a/grebi_summary_service/src/main/java/uk/ac/ebi/grebi_summary_service/GrebiSummarySvc.java b/grebi_summary_service/src/main/java/uk/ac/ebi/grebi_summary_service/GrebiSummarySvc.java new file mode 100644 index 0000000..5f5c74c --- /dev/null +++ b/grebi_summary_service/src/main/java/uk/ac/ebi/grebi_summary_service/GrebiSummarySvc.java @@ -0,0 +1,41 @@ +package uk.ac.ebi.grebi_summary_service; + +import com.google.gson.Gson; +import com.google.gson.JsonElement; +import com.google.gson.JsonParser; +import io.javalin.Javalin; +import io.javalin.http.Context; + +import java.io.*; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.HashMap; +import java.util.List; +import java.util.Map; + +public class GrebiSummarySvc { + private static Map jsons = new HashMap<>(); + + public static void main(String[] args) throws FileNotFoundException { + + Gson gson = new Gson(); + + var files = Arrays.stream(new File(System.getenv("GREBI_SUMMARY_JSON_SEARCH_PATH")).listFiles()).filter(File::isFile).filter(f -> f.getName().endsWith("_summary.json")).toArray(File[]::new); + + for (File f : files) { + var subgraph = f.getName().split("_summary.json")[0]; + jsons.put(subgraph, gson.fromJson(new InputStreamReader(new FileInputStream(f)), JsonElement.class)); + System.out.println("Loaded summary JSON for subgraph " + subgraph + " from " + f.getAbsolutePath()); + } + + Javalin app = Javalin.create(config -> { + }).start(8081); + + app.get("/", ctx -> { + ctx.contentType("application/json"); + ctx.result(gson.toJson(jsons)); + }); + } + +} + diff --git a/grebi_ui/src/App.tsx b/grebi_ui/src/App.tsx index c22742b..2a9bee7 100644 --- a/grebi_ui/src/App.tsx +++ b/grebi_ui/src/App.tsx @@ -44,9 +44,8 @@ class App extends React.Component { } /> } /> - } /> - - } /> + } /> + } /> } /> diff --git a/grebi_ui/src/components/ClassExpression.tsx b/grebi_ui/src/components/ClassExpression.tsx index 424cbdf..740d22e 100644 --- a/grebi_ui/src/components/ClassExpression.tsx +++ b/grebi_ui/src/components/ClassExpression.tsx @@ -6,9 +6,11 @@ import encodeNodeId from "../encodeNodeId"; import { Link } from "react-router-dom"; export default function ClassExpression({ + subgraph, node, expr }: { + subgraph:string, node:GraphNode|undefined, expr: any; }) { @@ -17,7 +19,7 @@ export default function ClassExpression({ if (typeof expr !== "object") { let mapped_value = node?.getRefs().get(expr); if(mapped_value) { - return {mapped_value.name} + return {mapped_value.name} } else { return expr } diff --git a/grebi_ui/src/components/SearchBox.tsx b/grebi_ui/src/components/SearchBox.tsx index 0586266..fbc39e3 100644 --- a/grebi_ui/src/components/SearchBox.tsx +++ b/grebi_ui/src/components/SearchBox.tsx @@ -17,10 +17,12 @@ interface SearchBoxEntry { } export default function SearchBox({ + subgraph, initialQuery, placeholder, collectionId, }: { + subgraph:string, initialQuery?: string; placeholder?: string; collectionId?: string; @@ -78,7 +80,7 @@ export default function SearchBox({ const [nodes, autocomplete] = await Promise.all([ getPaginated( - `api/v1/search?${new URLSearchParams({ + `api/v1/subgraphs/${subgraph}/search?${new URLSearchParams({ q: query, size: "5", lang: "en", @@ -90,7 +92,7 @@ export default function SearchBox({ ), showSuggestions ? get( - `api/v1/suggest?${new URLSearchParams({ + `api/v1/subgraphs/${subgraph}/suggest?${new URLSearchParams({ q: query, exactMatch: exact.toString(), includeObsoleteEntries: obsolete.toString(), diff --git a/grebi_ui/src/model/GraphNode.ts b/grebi_ui/src/model/GraphNode.ts index 8bc0c09..c8e21ff 100644 --- a/grebi_ui/src/model/GraphNode.ts +++ b/grebi_ui/src/model/GraphNode.ts @@ -31,12 +31,16 @@ export default class GraphNode { return PropVal.arrFrom(this.getDescriptions())[0]?.value } + getSubgraph():string { + return this.props['grebi:subgraph'] + } + getNodeId():string { return this.props['grebi:nodeId'] } getLinkUrl():string { - return `/nodes/${encodeNodeId(this.getNodeId())}`; + return `/subgraphs/${this.getSubgraph()}/nodes/${encodeNodeId(this.getNodeId())}`; } getId():PropVal { diff --git a/grebi_ui/src/pages/home/Home.tsx b/grebi_ui/src/pages/home/Home.tsx index 7388554..208bd51 100644 --- a/grebi_ui/src/pages/home/Home.tsx +++ b/grebi_ui/src/pages/home/Home.tsx @@ -6,16 +6,25 @@ import React, { Fragment } from "react"; import SearchBox from "../../components/SearchBox"; import { get } from "../../app/api"; import Stats from "../../model/Stats"; +import { MenuItem, Select } from "@mui/material"; export default function Home() { document.title = "EMBL-EBI Knowledge Graph"; let [stats, setStats] = useState(null); + let [subgraphs, setSubgraphs] = useState(null); + let [subgraph, setSubgraph] = useState(null); useEffect(() => { get("api/v1/stats").then(r => setStats(r)); }, []); + useEffect(() => { + get("api/v1/subgraphs").then(r => { + setSubgraphs(r) + setSubgraph(r[0]) + }); + }, []); return (
@@ -27,18 +36,37 @@ export default function Home() {
Welcome to the EMBL-EBI Knowledge Graph
-
- -
+ {subgraphs && subgraph ? + +
+ +
+
+ +
+ + : +
+ Loading graphs... +
+ }
Examples:  - + diabetes ,  - + BRCA1 diff --git a/grebi_ui/src/pages/node/NodePage.tsx b/grebi_ui/src/pages/node/NodePage.tsx index e60467e..f14f70e 100644 --- a/grebi_ui/src/pages/node/NodePage.tsx +++ b/grebi_ui/src/pages/node/NodePage.tsx @@ -11,7 +11,6 @@ import React from "react"; import LoadingOverlay from "../../components/LoadingOverlay"; import GraphNode from "../../model/GraphNode"; import { get } from "../../app/api"; -import NodeProperties from "./NodeProperties"; import { AccountTree, Share, Visibility, VisibilityOff } from "@mui/icons-material"; import Header from "../../components/Header"; import LanguagePicker from "../../components/LanguagePicker"; @@ -29,6 +28,7 @@ import PropTable from "./prop_table/PropTable"; export default function NodePage() { const params = useParams(); const [searchParams, setSearchParams] = useSearchParams(); + const subgraph: string = params.subgraph as string; const nodeId: string = atob(params.nodeId as string); const lang = searchParams.get("lang") || "en"; @@ -49,7 +49,7 @@ export default function NodePage() { useEffect(() => { async function getNode() { - let graphNode = new GraphNode(await get(`api/v1/nodes/${nodeId}?lang=${lang}`)) + let graphNode = new GraphNode(await get(`api/v1/subgraphs/${subgraph}/nodes/${nodeId}?lang=${lang}`)) setNode(graphNode) let datasources = graphNode.getDatasources(); @@ -79,7 +79,7 @@ export default function NodePage() { {pageDesc && }
- +
{pageTitle} { node.extractType()?.long && {node.extractType()?.long}} diff --git a/grebi_ui/src/pages/node/prop_table/PropVals.tsx b/grebi_ui/src/pages/node/prop_table/PropVals.tsx index 6f93dd3..38b1f3a 100644 --- a/grebi_ui/src/pages/node/prop_table/PropVals.tsx +++ b/grebi_ui/src/pages/node/prop_table/PropVals.tsx @@ -40,9 +40,9 @@ export default function PropVals(params:{ node:GraphNode,prop:string,values:Prop } -function PropValue(params:{node:GraphNode,prop:string,value:PropVal,monospace:boolean,separator:string}) { +function PropValue(params:{subgraph:string,node:GraphNode,prop:string,value:PropVal,monospace:boolean,separator:string}) { - let { node, prop, value, monospace, separator } = params; + let { subgraph, node, prop, value, monospace, separator } = params; if(typeof value.value === 'object') { if(value.value["rdf:type"] !== undefined) { @@ -58,7 +58,7 @@ function PropValue(params:{node:GraphNode,prop:string,value:PropVal,monospace:bo if(mapped_value) { return ( - {separator} { + {separator} { (mapped_value.name && pickBestDisplayName(mapped_value.name)) || value.value } diff --git a/grebi_ui/src/pages/search/Search.tsx b/grebi_ui/src/pages/search/Search.tsx index 2ed7d42..a6ab291 100644 --- a/grebi_ui/src/pages/search/Search.tsx +++ b/grebi_ui/src/pages/search/Search.tsx @@ -1,6 +1,6 @@ import { Close, KeyboardArrowDown } from "@mui/icons-material"; import { useCallback, useEffect, useState } from "react"; -import { Link, useSearchParams } from "react-router-dom"; +import { Link, useParams, useSearchParams } from "react-router-dom"; import { copyToClipboard, randomString, usePrevious } from "../../app/util"; import Header from "../../components/Header"; import LoadingOverlay from "../../components/LoadingOverlay"; @@ -16,6 +16,9 @@ export default function Search() { const [searchParams] = useSearchParams(); const search = searchParams.get("q") || ""; + const params = useParams(); + const subgraph: string = params.subgraph as string; + let [loadingResults, setLoadingResults] = useState(false); let [results, setResults] = useState([]); let [totalResults, setTotalResults] = useState(0); @@ -93,7 +96,7 @@ export default function Search() { useEffect(() => { async function doSearch() { - let res = (await getPaginated('api/v1/search', { + let res = (await getPaginated(`api/v1/subgraphs/${subgraph}/search`, { page: page.toString(), size: rowsPerPage.toString(), q: search, facet: ['grebi:datasources','grebi:type'] /*grebi__datasource: datasourceFacetselected, @@ -123,7 +126,7 @@ export default function Search() {
- +