Skip to content

Commit 4887506

Browse files
hugomanguinhashugomanguinhas
committed
Couple of fixes, improved migration code to support FTP
1 parent cf56bfe commit 4887506

File tree

32 files changed

+430
-134
lines changed

32 files changed

+430
-134
lines changed

‎record-api-common/src/main/java/eu/europeana/api/config/AppConfigConstants.java‎

Lines changed: 7 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -7,11 +7,9 @@ public class AppConfigConstants {
77
public static final String BEAN_RECORD_DATA_STORE = "recordDataStore";
88
public static final String BEAN_RECORD_REPO = "recordRepo";
99
public static final String BEAN_RECORD_SERVICE = "recordService";
10-
public static final String BEAN_RECORD_JSONLD_SERIALIZER = "recordJsonldWriter";
11-
public static final String BEAN_RECORD_XML_SERIALIZER = "recordXmlWriter";
1210

1311
// v2 beans
14-
public static final String BEAN_RECORD_V2_JSON_SERIALIZER = "recordJsonV2Writer";
12+
public static final String BEAN_FORMAT_WRITER_V2_JSON = "recordJsonV2Writer";
1513

1614

1715
// serialiser beans
@@ -23,9 +21,12 @@ public class AppConfigConstants {
2321
public static final String BEAN_NAMESPACE_RESOLVER= "namespaceResolver";
2422
public static final String BEAN_DEFAULT_URI_RESOLVER= "defaultUriResolver";
2523
public static final String BEAN_RECORD_TEMPLATE_LIBRARY = "recordApiTemplateLibrary";
26-
public static final String BEAN_JENA_FORAMAT_WRITER_TURTLE = "jenaFormatWriterTurtle";
27-
public static final String BEAN_JENA_FORAMAT_WRITER_N3 = "jenaFormatWriterN3";
28-
public static final String BEAN_JENA_FORAMAT_WRITER_NT = "jenaFormatWriterNt";
24+
25+
public static final String BEAN_FORMAT_WRITER_TURTLE = "recordWriterTurtle";
26+
public static final String BEAN_FORMAT_WRITER_N3 = "recordWriterN3";
27+
public static final String BEAN_FORMAT_WRITER_NT = "recordWriterNt";
28+
public static final String BEAN_FORMAT_WRITER_JSONLD = "recordWriterJsonLD";
29+
public static final String BEAN_FORMAT_WRITER_XML = "recordWriterXml";
2930

3031

3132
// media config beans

‎record-api-migration/pom.xml‎

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -53,6 +53,11 @@
5353
<artifactId>jackson-datatype-jsr310</artifactId>
5454
<version>${jackson.version}</version>
5555
</dependency>
56+
<dependency>
57+
<groupId>commons-net</groupId>
58+
<artifactId>commons-net</artifactId>
59+
<version>3.10.0</version>
60+
</dependency>
5661
</dependencies>
5762

5863
</project>

‎record-api-migration/src/main/java/eu/europeana/api/record/migration/FTPSupport.java‎

Lines changed: 120 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,120 @@
1+
/**
2+
*
3+
*/
4+
package eu.europeana.api.record.migration;
5+
6+
import java.io.IOException;
7+
import java.io.InputStream;
8+
import java.net.SocketException;
9+
import java.net.URI;
10+
import java.net.URL;
11+
12+
import org.apache.commons.net.ftp.FTPClient;
13+
import org.apache.commons.net.ftp.FTPFile;
14+
15+
/**
16+
* @author Hugo
17+
* @since 22 May 2024
18+
*/
19+
public class FTPSupport
20+
{
21+
private FTPClient client;
22+
private String baseURL;
23+
private URI uri;
24+
25+
public FTPSupport(String url) throws SocketException, IOException {
26+
uri = URI.create(url);
27+
client = new FTPClient();
28+
client.connect(uri.getHost(), uri.getPort());
29+
boolean isSuccess = client.login(uri.getUserInfo(), "");
30+
if ( !isSuccess ) { throw new IOException("Cannot connnect"); }
31+
client.enterLocalPassiveMode();
32+
33+
client.changeWorkingDirectory(uri.getPath());
34+
baseURL = url.replace(uri.getPath(), "");
35+
}
36+
37+
public FTPContext getRoot() throws IOException {
38+
String path = uri.getPath();
39+
40+
FTPFile[] files = client.listFiles(path);
41+
if ( files.length == 1 ) {
42+
return new FTPContext(path, files[0]);
43+
}
44+
return new FTPRoot(path, files);
45+
}
46+
47+
public void close() throws IOException {
48+
try {
49+
client.logout();
50+
}
51+
finally {
52+
client.disconnect();
53+
}
54+
}
55+
56+
public class FTPContext {
57+
58+
private String path;
59+
private FTPFile file;
60+
61+
public FTPContext(String path, FTPFile file) {
62+
this.path = path;
63+
this.file = file;
64+
}
65+
66+
public FTPContext newContext(FTPFile file) {
67+
return new FTPContext(this.path + "/" + file.getName(), file);
68+
}
69+
70+
public FTPFile[] listFiles() throws IOException {
71+
return client.listFiles(path);
72+
}
73+
74+
public boolean isDirectory() {
75+
return file.isDirectory();
76+
}
77+
78+
public String getURL() {
79+
return baseURL + path;
80+
}
81+
82+
public FTPFile getFile() {
83+
return file;
84+
}
85+
86+
public String getFilename() {
87+
return file.getName();
88+
}
89+
90+
public InputStream openStream() throws IOException {
91+
return client.retrieveFileStream(this.path);
92+
}
93+
}
94+
95+
public class FTPRoot extends FTPContext {
96+
97+
private FTPFile[] files;
98+
99+
public FTPRoot(String path, FTPFile[] files) {
100+
super(path, null);
101+
this.files = files;
102+
}
103+
104+
public FTPFile[] listFiles() throws IOException {
105+
return files;
106+
}
107+
108+
public boolean isDirectory() {
109+
return true;
110+
}
111+
112+
public FTPFile getFile() {
113+
return null;
114+
}
115+
116+
public String getFilename() {
117+
return null;
118+
}
119+
}
120+
}

‎record-api-migration/src/main/java/eu/europeana/api/record/migration/MigrationHandler.java‎

Lines changed: 6 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -19,6 +19,7 @@
1919
import org.apache.jena.rdfxml.xmlinput.DOM2Model;
2020
import org.apache.jena.vocabulary.RDF;
2121
import org.springframework.beans.factory.annotation.Autowired;
22+
import org.springframework.beans.factory.annotation.Qualifier;
2223
import org.springframework.stereotype.Service;
2324
import org.w3c.dom.Document;
2425
import org.xml.sax.SAXParseException;
@@ -33,6 +34,7 @@
3334
import java.util.concurrent.TimeUnit;
3435

3536
import static org.apache.jena.rdf.model.ResourceFactory.createResource;
37+
import static eu.europeana.api.config.AppConfigConstants.*;
3638

3739
/**
3840
* @author Hugo
@@ -66,7 +68,10 @@ public class MigrationHandler {
6668
private final TemplateLibrary library;
6769

6870
@Autowired
69-
public MigrationHandler(MigrationSettings settings, FormatHandlerRegistry registry, MigrationRepository migrationRepository, TemplateLibrary library) {
71+
public MigrationHandler(MigrationSettings settings
72+
, FormatHandlerRegistry registry
73+
, MigrationRepository migrationRepository
74+
, @Qualifier(BEAN_RECORD_TEMPLATE_LIBRARY) TemplateLibrary library) {
7075
this.settings = settings;
7176
this.registry = registry;
7277
this.migrationRepository = migrationRepository;

‎record-api-migration/src/main/java/eu/europeana/api/record/migration/MigrationSettings.java‎

Lines changed: 31 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -15,23 +15,48 @@
1515
import eu.europeana.jena.encoder.library.DefaultUriNormalizer;
1616
import eu.europeana.jena.encoder.library.TemplateLibrary;
1717
import jakarta.annotation.Resource;
18-
import org.springframework.beans.factory.annotation.Value;
18+
19+
import org.springframework.beans.factory.annotation.Qualifier;
1920
import org.springframework.context.annotation.Bean;
2021
import org.springframework.context.annotation.Configuration;
2122
import org.springframework.context.annotation.Import;
2223
import org.springframework.context.annotation.PropertySource;
2324

25+
import com.fasterxml.jackson.databind.ObjectMapper;
26+
27+
import static eu.europeana.api.config.AppConfigConstants.*;
28+
29+
2430
@Configuration
2531
@PropertySource(
2632
value = {"classpath:migration.properties", "classpath:migration.user.properties"},
2733
ignoreResourceNotFound = true)
28-
@Import({MediaTypeConfig.class, DataSourceConfig.class})
34+
@Import({ MediaTypeConfig.class, DataSourceConfig.class
35+
, RecordIOConfig.class, JsonLdWriter.class, XmlRecordWriter.class
36+
, RecordApiTemplateLibrary.class, FormatHandlerRegistry.class})
2937
public class MigrationSettings {
3038

3139
@Resource(name = AppConfigConstants.BEAN_MEDIA_TYPES)
3240
private MediaTypes mediaTypes;
3341

42+
@Resource( name = BEAN_JSON_MAPPER)
43+
private ObjectMapper mapper;
44+
45+
/*
3446
@Bean
47+
public FormatHandlerRegistry getFormatHandlerRegistry(
48+
@Qualifier(BEAN_FORMAT_WRITER_JSONLD) JsonLdWriter jsonLdWriter
49+
, @Qualifier(BEAN_FORMAT_WRITER_XML) XmlRecordWriter xmlRecordWriter
50+
, @Qualifier(BEAN_FORMAT_WRITER_TURTLE) JenaBasedFormatWriter jenaBasedTurtleWriter
51+
, @Qualifier(BEAN_FORMAT_WRITER_N3) JenaBasedFormatWriter jenaBasedN3Writer
52+
, @Qualifier(BEAN_FORMAT_WRITER_NT) JenaBasedFormatWriter jenaBasedNTWriter) {
53+
return new FormatHandlerRegistry(
54+
jsonLdWriter,
55+
xmlRecordWriter,
56+
jenaBasedTurtleWriter,
57+
jenaBasedN3Writer,
58+
jenaBasedNTWriter);
59+
}
3560
public FormatHandlerRegistry getFormatHandlerRegistry() {
3661
return new FormatHandlerRegistry(
3762
new JsonLdWriter(),
@@ -43,12 +68,13 @@ public FormatHandlerRegistry getFormatHandlerRegistry() {
4368
4469
@Bean
4570
public TemplateLibrary getTemplateLibrary() {
46-
return new RecordApiTemplateLibrary(new CodecRegistry(), new Namespaces(), new DefaultUriNormalizer());
71+
return new RecordApiTemplateLibrary(
72+
new CodecRegistry(), new Namespaces()
73+
, new DefaultUriNormalizer(), mediaTypes);
4774
}
75+
*/
4876

4977
public MediaTypes getMediaTypes() {
5078
return mediaTypes;
5179
}
52-
53-
5480
}

‎record-api-migration/src/main/java/eu/europeana/api/record/migration/RecordJenaProcessor.java‎

Lines changed: 62 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -2,13 +2,16 @@
22

33
import static org.apache.jena.rdf.model.ResourceFactory.*;
44

5+
import java.net.URLEncoder;
6+
import java.nio.charset.Charset;
57
import java.time.OffsetDateTime;
68
import java.time.format.DateTimeFormatter;
79
import java.util.*;
810

9-
import eu.europeana.api.config.AppConfigConstants;
1011
import eu.europeana.api.model.MediaType;
1112
import eu.europeana.api.model.MediaTypes;
13+
14+
import org.apache.jena.datatypes.xsd.XSDDatatype;
1215
import org.apache.jena.rdf.model.Literal;
1316
import org.apache.jena.rdf.model.Model;
1417
import org.apache.jena.rdf.model.Property;
@@ -18,7 +21,6 @@
1821
import org.apache.jena.rdf.model.Statement;
1922
import org.apache.jena.rdf.model.StmtIterator;
2023
import org.apache.jena.sparql.vocabulary.FOAF;
21-
import org.apache.jena.util.ResourceUtils;
2224
import org.apache.jena.vocabulary.DC;
2325
import org.apache.jena.vocabulary.DCTerms;
2426
import org.apache.jena.vocabulary.OWL;
@@ -39,7 +41,6 @@
3941
import eu.europeana.api.edm.SVCS;
4042
import eu.europeana.api.edm.XSD;
4143
*/
42-
import eu.europeana.api.record.io.jena.RecordApiTemplateLibrary;
4344
import eu.europeana.jena.edm.CC;
4445
import eu.europeana.jena.edm.EBUCORE;
4546
import eu.europeana.jena.edm.EDM;
@@ -59,6 +60,8 @@ public class RecordJenaProcessor {
5960
private static DateTimeFormatter DATETIME_FORMAT
6061
= DateTimeFormatter.ofPattern("uuuu-MM-dd'T'HH:mm:ss'Z'");
6162

63+
private static String THUMBNAIL_URL = "https://api.europeana.eu/thumbnail/v2/url.json?uri=%s&type=%s&size=%s";
64+
6265
/*
6366
private static final Resource ProvidedCHO = createResource(EDM.NS + EDM.ProvidedCHO);
6467
private static final Resource Proxy = createResource(ORE.NS + ORE.Proxy);
@@ -150,6 +153,9 @@ public Resource upgrade(Resource cho) {
150153
fixWebResourcerReference(getObjects(m.listStatements(null, EDM.isShownAt, (RDFNode)null)));
151154
fixWebResourcerReference(getObjects(m.listStatements(null, EDM.hasView, (RDFNode)null)));
152155

156+
generateThumbnails(getObjects(m.listStatements(null, EDM.isShownBy, (RDFNode)null)));
157+
generateThumbnails(getObjects(m.listStatements(null, EDM.hasView, (RDFNode)null)));
158+
153159
removeLooseResources(getAsList(m.listResourcesWithProperty(RDF.type, SVCS.Service)));
154160
removeLooseResources(getAsList(m.listResourcesWithProperty(RDF.type, CC.License)));
155161

@@ -258,6 +264,51 @@ private void cleanTechMetadata(List<Resource> resources)
258264
}
259265
}
260266

267+
private Dimension getSize(Resource r) {
268+
Integer width = getAsInteger(r.getProperty(EBUCORE.width));
269+
Integer height = getAsInteger(r.getProperty(EBUCORE.height));
270+
return ( width != null && height != null ? new Dimension(width,height) : null );
271+
}
272+
273+
private Integer getAsInteger(Statement stmt) {
274+
return ( stmt == null ? null : stmt.getInt() );
275+
}
276+
277+
private void generateThumbnails(List<Resource> resources) {
278+
String thumbURL;
279+
for ( Resource r : resources ) {
280+
Statement stmt = r.getProperty(EBUCORE.hasMimeType);
281+
if ( stmt == null ) { continue; }
282+
283+
Optional<MediaType> mediaType = mediaTypes.getMediaType(stmt.getString());
284+
if ( mediaType.isEmpty() ) { continue; }
285+
286+
MediaType mt = mediaType.get();
287+
if ( !mt.isBrowserSupported() ) { continue; }
288+
289+
Model m = r.getModel();
290+
String url = URLEncoder.encode(r.getURI(), Charset.defaultCharset());
291+
Dimension size = getSize(r);
292+
if ( size == null ) { continue; }
293+
294+
thumbURL = String.format(THUMBNAIL_URL, url, mt.getType(), "w400");
295+
r.addProperty(EDM.preview
296+
, generateThumbnailResource(m.getResource(thumbURL), mt, size.resizeToWidth(400)));
297+
298+
thumbURL = String.format(THUMBNAIL_URL, url, mt.getType(), "w200");
299+
r.addProperty(EDM.preview
300+
, generateThumbnailResource(m.getResource(thumbURL), mt, size.resizeToWidth(200)));
301+
}
302+
}
303+
304+
private Resource generateThumbnailResource(Resource r, MediaType mt, Dimension size) {
305+
r.addLiteral(EDM.type, mt.getType());
306+
r.addProperty(EBUCORE.width, Integer.toString(size.width), XSDDatatype.XSDinteger);
307+
r.addProperty(EBUCORE.height, Integer.toString(size.height), XSDDatatype.XSDinteger);
308+
r.addLiteral(EBUCORE.hasMimeType, "image/jpeg");
309+
return r;
310+
}
311+
261312
private void fixWebResourcerReference(List<Resource> resources)
262313
{
263314
for ( Resource r : resources ) {
@@ -717,4 +768,12 @@ private int getOrder(Resource r) {
717768
return 2;
718769
}
719770
}
771+
772+
private static record Dimension(int width, int height) {
773+
774+
public Dimension resizeToWidth(int width) {
775+
float ratio = (float)this.height / this.width;
776+
return new Dimension(width, Math.round(ratio * width));
777+
}
778+
}
720779
}

0 commit comments

Comments
 (0)