Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

add support for gzipped ontologies (#761) #762

Open
wants to merge 3 commits into
base: dev
Choose a base branch
from
Open

add support for gzipped ontologies (#761) #762

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
3 commits
Select commit Hold shift + click to select a range
a754d90
add support for gzipped ontologies (#761)
jamesamcl Oct 13, 2024
106d68c
add progress printouts to reified property annotator
jamesamcl Oct 19, 2024
a57f19f
Trigger build pipeline
haideriqbal Dec 11, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
12 changes: 11 additions & 1 deletion ...ad/predownloader/src/main/java/uk/ac/ebi/ols4/predownloader/OntologyDownloaderThread.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -21,6 +21,7 @@
import java.util.Set;
import java.util.function.Consumer;
import java.util.function.Function;
import java.util.zip.GZIPInputStream;

import org.apache.commons.io.FileUtils;
import org.apache.commons.io.input.TeeInputStream;
Expand Down Expand Up @@ -72,12 +73,21 @@ public void run() {
String mimetype = downloadURL(ontologyUrl, path);

Lang lang = RDFLanguages.contentTypeToLang(mimetype);
if(lang == null) {
lang = RDFLanguages.filenameToLang(ontologyUrl, Lang.RDFXML);
}
if(lang == null) {
lang = Lang.RDFXML;
}

InputStream is = new FileInputStream(path);

if(ontologyUrl.endsWith(".gz")) {
is = new GZIPInputStream(is);
}

// parse to look for imports only
createParser(lang).source(new FileInputStream(path)).parse(new StreamRDF() {
createParser(lang).source(is).parse(new StreamRDF() {
public void start() {}
public void quad(Quad quad) {}
public void base(String base) {}
Expand Down
111 changes: 73 additions & 38 deletions dataload/rdf2json/src/main/java/uk/ac/ebi/rdf2json/OntologyGraph.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -8,6 +8,13 @@
import uk.ac.ebi.rdf2json.properties.*;

import org.apache.jena.riot.Lang;
import org.apache.http.Header;
import org.apache.http.HttpEntity;
import org.apache.http.HttpResponse;
import org.apache.http.client.config.RequestConfig;
import org.apache.http.client.methods.HttpGet;
import org.apache.http.impl.client.CloseableHttpClient;
import org.apache.http.impl.client.HttpClientBuilder;
import org.apache.jena.graph.Node;
import org.apache.jena.graph.Triple;
import org.apache.jena.riot.RDFParser;
Expand All @@ -16,13 +23,19 @@
import org.apache.jena.sparql.core.Quad;

import java.io.IOException;
import java.io.InputStream;
import java.io.File;
import java.io.FileInputStream;
import java.io.FileNotFoundException;
import java.io.FileOutputStream;
import java.nio.charset.StandardCharsets;
import java.nio.file.Files;
import java.nio.file.Paths;
import java.nio.file.StandardOpenOption;
import java.util.*;
import java.util.stream.Collectors;
import java.util.zip.GZIPInputStream;

import static uk.ac.ebi.rdf2json.OntologyNode.NodeType.*;
import static uk.ac.ebi.ols.shared.DefinedFields.*;

Expand Down Expand Up @@ -56,45 +69,67 @@ private RDFParserBuilder createParser(Lang lang) {
}
}

private void parseRDF(String url) {
private void parseRDF(String url, InputStream is, String contentType) throws IOException {
if(url.endsWith(".gz")) {
System.out.println("parseRDF: Decompressing gzipped ontology " + url);
is = new GZIPInputStream(is);
url = url.substring(0, url.length() - 3);
}

try {
if (loadLocalFiles && !url.contains("://")) {
logger.debug("Using local file for {}", url);
sourceFileTimestamp = new File(url).lastModified();
createParser(RDFLanguages.filenameToLang(url, Lang.RDFXML))
.source(new FileInputStream(url)).parse(this);
} else {
if (downloadedPath != null) {
String existingDownload = downloadedPath + "/" + urlToFilename(url);
try {
FileInputStream is = new FileInputStream(existingDownload);
logger.debug("Using predownloaded file for {}", url);
sourceFileTimestamp = new File(existingDownload).lastModified();
Lang lang = null;
try {
String existingDownloadMimeType = Files.readString(Paths.get(existingDownload + ".mimetype"));
lang = RDFLanguages.contentTypeToLang(existingDownloadMimeType);
} catch(IOException ignored) {
}
if(lang == null) {
lang = Lang.RDFXML;
}
createParser(lang).source(is).parse(this);
} catch (Exception e) {
logger.error("Downloading (not predownloaded) {}", url);
sourceFileTimestamp = System.currentTimeMillis();
createParser(null).source(url).parse(this);
}
} else {
logger.debug("Downloading (no predownload path provided) {}", url);
sourceFileTimestamp = System.currentTimeMillis();
createParser(null).source(url).parse(this);
}
}
} catch (FileNotFoundException e) {
throw new RuntimeException(e);
Lang lang = null;
if(contentType != null) {
lang = RDFLanguages.contentTypeToLang(contentType);
}
if(lang == null) {
lang = RDFLanguages.filenameToLang(url, Lang.RDFXML);
}
if(lang == null) {
lang = Lang.RDFXML;
}

createParser(lang).source(is).parse(this);
}

private void parseRDF(String url) throws IOException {

if (loadLocalFiles && !url.contains("://")) {
logger.debug("parseRDF: Using local file for {}", url);
sourceFileTimestamp = new File(url).lastModified();
parseRDF(url, new FileInputStream(url), null);
return;
}

if (downloadedPath != null) {
String existingDownload = downloadedPath + "/" + urlToFilename(url);
InputStream is = new FileInputStream(existingDownload);
logger.debug("parseRDF: Using predownloaded file for {}", url);
sourceFileTimestamp = new File(existingDownload).lastModified();
String existingDownloadMimeType = Files.readString(Paths.get(existingDownload + ".mimetype"));
parseRDF(url, is, existingDownloadMimeType);
return;
}

logger.error("parseRDF: Downloading (not predownloaded) {}", url);
sourceFileTimestamp = System.currentTimeMillis();

HttpEntity res = getURL(url);
InputStream is = res.getContent();
String contentType = res.getContentType().getValue();
parseRDF(url, is, contentType);
}

private static HttpEntity getURL(String url) throws FileNotFoundException, IOException {

RequestConfig config = RequestConfig.custom()
.setConnectTimeout(5000)
.setConnectionRequestTimeout(5000)
.setSocketTimeout(5000).build();

CloseableHttpClient client = HttpClientBuilder.create().setDefaultRequestConfig(config).build();

HttpGet request = new HttpGet(url);
HttpResponse response = client.execute(request);
return response.getEntity();
}

private String urlToFilename(String url) {
Expand All @@ -107,7 +142,7 @@ private String urlToFilename(String url) {
String downloadedPath;


OntologyGraph(Map<String, Object> config, boolean loadLocalFiles, boolean noDates, String downloadedPath) {
OntologyGraph(Map<String, Object> config, boolean loadLocalFiles, boolean noDates, String downloadedPath) throws IOException {

this.loadLocalFiles = loadLocalFiles;
this.downloadedPath = downloadedPath;
Expand Down
16 changes: 14 additions & 2 deletions dataload/rdf2json/src/main/java/uk/ac/ebi/rdf2json/annotators/ReifiedPropertyAnnotator.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -9,6 +9,7 @@
import uk.ac.ebi.rdf2json.properties.PropertyValueURI;

import java.util.List;
import java.util.stream.Collectors;

public class ReifiedPropertyAnnotator {

Expand All @@ -17,8 +18,19 @@ public class ReifiedPropertyAnnotator {
public static void annotateReifiedProperties(OntologyGraph graph) {

long startTime3 = System.nanoTime();
for(String id : graph.nodes.keySet()) {
OntologyNode c = graph.nodes.get(id);

var nodes = graph.nodes.keySet().stream()
.map((String id) -> { return graph.nodes.get(id); })
.filter(c -> c.types.contains(OntologyNode.NodeType.AXIOM))
.collect(Collectors.toList());

logger.info("ReifiedPropertyAnnotator: processing {} axiom nodes", nodes.size());
int n = 0;
for(OntologyNode c : nodes) {
++ n;
if(n % 1000 == 0) {
logger.info("ReifiedPropertyAnnotator: processed {} of {} axiom nodes", n, nodes.size());
}
if (c.types.contains(OntologyNode.NodeType.AXIOM)) {

PropertyValue source = c.properties.getPropertyValue("http://www.w3.org/2002/07/owl#annotatedSource");
Expand Down
Loading