Skip to content

Commit 47feaf5

Browse files
marcin-rpmarcin-rp
committed
Merge branch 'MET-6412-Unit-Test-and-test-coverage-for-Flink-solution' into develop
# Conflicts: # http/src/test/java/eu/europeana/processing/http/reader/AbstractUnpackingTest.java # http/src/test/java/eu/europeana/processing/http/reader/HttpEnumeratorTest.java # http/src/test/java/eu/europeana/processing/http/reader/HttpReaderTest.java
2 parents 39728ba + cda45fe commit 47feaf5

File tree

42 files changed

+1698
-20
lines changed

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

42 files changed

+1698
-20
lines changed

‎.github/workflows/ci.yml‎

Lines changed: 18 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -5,9 +5,21 @@ on: push
55
jobs:
66

77
ci:
8-
uses: europeana/metis-actions/.github/workflows/ci.yml@main
9-
with:
10-
sonar_organization: europeana
11-
sonar_project_key: europeana_metis-processing-engine
12-
secrets:
13-
SONAR_TOKEN: ${{ secrets.METIS_SONAR_TOKEN }}
8+
runs-on: ubuntu-latest
9+
steps:
10+
- name: Set METIS_PROCESSING_HTTP_JOBS_DIR env var
11+
run: |
12+
echo "METIS_PROCESSING_HTTP_JOBS_DIR=$RUNNER_TEMP/http-jobs" >> $GITHUB_ENV
13+
mkdir -p "$RUNNER_TEMP/http-jobs"
14+
15+
- name: Install media software
16+
uses: europeana/metis-actions/actions/media-software-install-steps@main
17+
- name: Build code and Sonar Analyze
18+
uses: europeana/metis-actions/actions/build-analyze@main
19+
with:
20+
sonar_organization: europeana
21+
sonar_project_key: europeana_metis-processing-engine
22+
sonar_token: ${{ secrets.METIS_SONAR_TOKEN }}
23+
github_token: ${{ secrets.GITHUB_TOKEN }}
24+
- name: Upload all jars to target-jars
25+
uses: europeana/metis-actions/actions/upload-target-jars@main

‎enrichment/pom.xml‎

Lines changed: 12 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -78,6 +78,18 @@
7878
</exclusion>
7979
</exclusions>
8080
</dependency>
81+
<dependency>
82+
<groupId>eu.europeana.metis-processing-engine</groupId>
83+
<artifactId>test-commons</artifactId>
84+
<version>${parent.version}</version>
85+
<scope>test</scope>
86+
</dependency>
87+
<dependency>
88+
<groupId>org.wiremock</groupId>
89+
<artifactId>wiremock-standalone</artifactId>
90+
<version>${wiremock.version}</version>
91+
<scope>test</scope>
92+
</dependency>
8193
</dependencies>
8294

8395
<build>

‎enrichment/src/test/java/eu/europeana/processing/enrichment/EnrichmentJobTest.java‎

Lines changed: 94 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,94 @@
1+
package eu.europeana.processing.enrichment;
2+
3+
import static com.github.tomakehurst.wiremock.client.WireMock.aResponse;
4+
import static com.github.tomakehurst.wiremock.client.WireMock.get;
5+
import static com.github.tomakehurst.wiremock.client.WireMock.urlEqualTo;
6+
import static com.github.tomakehurst.wiremock.core.WireMockConfiguration.wireMockConfig;
7+
8+
import com.github.tomakehurst.wiremock.WireMockServer;
9+
import com.github.tomakehurst.wiremock.common.ConsoleNotifier;
10+
import com.github.tomakehurst.wiremock.http.JvmProxyConfigurer;
11+
import eu.europeana.processing.job.JobParamName;
12+
import eu.europeana.processing.test.common.AbstractJobTest;
13+
import java.io.IOException;
14+
import java.io.InputStream;
15+
import org.junit.jupiter.api.BeforeAll;
16+
import org.junit.jupiter.api.Disabled;
17+
import org.junit.jupiter.api.Test;
18+
import org.junit.jupiter.api.extension.ExtendWith;
19+
import org.mockito.junit.jupiter.MockitoExtension;
20+
import org.springframework.http.HttpStatus;
21+
22+
23+
@Disabled("It was not finished and will be finished as part of ticket: MET-6675")
24+
@ExtendWith(MockitoExtension.class)
25+
class EnrichmentJobTest extends AbstractJobTest {
26+
27+
private static WireMockServer wireMockServer;
28+
29+
@BeforeAll
30+
static void createWireMock() throws IOException {
31+
wireMockServer = new WireMockServer(wireMockConfig()
32+
.dynamicPort()
33+
.enableBrowserProxying(true)
34+
.notifier(new ConsoleNotifier(true)));
35+
wireMockServer.start();
36+
JvmProxyConfigurer.configureFor(wireMockServer);
37+
mockEnrichmentWeb();
38+
}
39+
40+
@Test
41+
void shouldProperlyRunEnrichment() throws Exception {
42+
startPostgresDbServer("enrichment-input.sql");
43+
String[] args = prepareArgs(
44+
"--" + JobParamName.DEREFERENCE_SERVICE_URL, "http://localhost:" + wireMockServer.port() ,
45+
"--" + JobParamName.ENRICHMENT_ENTITY_MANAGEMENT_URL, "http://localhost:" + wireMockServer.port() + "/entitymgmt",
46+
"--" + JobParamName.ENRICHMENT_ENTITY_API_URL, "http://localhost:" + wireMockServer.port() + "/entity",
47+
"--" + JobParamName.ENRICHMENT_ENTITY_API_TOKEN_ENDPOINT, "entityApiGrantParams",
48+
"--" + JobParamName.ENRICHMENT_ENTITY_API_GRANT_PARAMS, "test-api-grant-params");
49+
50+
EnrichmentJob.main(args);
51+
52+
assertThatResultRowIsSavedInDb();
53+
assertNoErrorsSavedInDb();
54+
}
55+
56+
@Test
57+
void shouldSaveRecordErrorInDB() throws Exception {
58+
startPostgresDbServer("invalid-record-input.sql");
59+
String[] args = prepareArgs(
60+
"--" + JobParamName.DEREFERENCE_SERVICE_URL, "http://localhost:" + wireMockServer.port() ,
61+
"--" + JobParamName.ENRICHMENT_ENTITY_MANAGEMENT_URL, "http://localhost:" + wireMockServer.port() + "/entitymgmt",
62+
"--" + JobParamName.ENRICHMENT_ENTITY_API_URL, "http://localhost:" + wireMockServer.port() + "/entity",
63+
"--" + JobParamName.ENRICHMENT_ENTITY_API_TOKEN_ENDPOINT, "entityApiGrantParams",
64+
"--" + JobParamName.ENRICHMENT_ENTITY_API_GRANT_PARAMS, "test-api-grant-params");
65+
66+
EnrichmentJob.main(args);
67+
68+
assertThatErrorIsSavedInDb();
69+
}
70+
71+
@Override
72+
protected int stepNumber() {
73+
return 6;
74+
}
75+
76+
private static void mockEnrichmentWeb() throws IOException {
77+
wireMockServer.stubFor(
78+
get(urlEqualTo("/dereference?uri=http%3A%2F%2Fdata.europeana.eu%2Ftimespan%2F20"))
79+
.willReturn(aResponse()
80+
.withHeader("Content-Type", "application/json")
81+
.withBody(getResourceFileContent("entity-api/entity-api-response-timespan.json"))
82+
.withStatus(HttpStatus.OK.value())));
83+
84+
}
85+
86+
87+
88+
private static String getResourceFileContent(String fileName) throws IOException {
89+
try (InputStream resourceStream = EnrichmentJobTest.class.getClassLoader().getResourceAsStream(fileName)) {
90+
return new String(resourceStream.readAllBytes());
91+
}
92+
}
93+
94+
}

‎enrichment/src/test/resources/ennrichment-output.xml‎

Lines changed: 144 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,144 @@
1+
<?xml version="1.0" encoding="UTF-8"?>
2+
<rdf:RDF xmlns:rdf="http://www.w3.org/1999/02/22-rdf-syntax-ns#" xmlns:dc="http://purl.org/dc/elements/1.1/" xmlns:dcterms="http://purl.org/dc/terms/" xmlns:edm="http://www.europeana.eu/schemas/edm/" xmlns:owl="http://www.w3.org/2002/07/owl#" xmlns:wgs84_pos="http://www.w3.org/2003/01/geo/wgs84_pos#" xmlns:skos="http://www.w3.org/2004/02/skos/core#" xmlns:rdaGr2="http://rdvocab.info/ElementsGr2/" xmlns:foaf="http://xmlns.com/foaf/0.1/" xmlns:ebucore="http://www.ebu.ch/metadata/ontologies/ebucore/ebucore#" xmlns:doap="http://usefulinc.com/ns/doap#" xmlns:odrl="http://www.w3.org/ns/odrl/2/" xmlns:cc="http://creativecommons.org/ns#" xmlns:ore="http://www.openarchives.org/ore/terms/" xmlns:svcs="http://rdfs.org/sioc/services#" xmlns:oa="http://www.w3.org/ns/oa#" xmlns:dqv="http://www.w3.org/ns/dqv#">
3+
<edm:ProvidedCHO rdf:about="/one-recor/item_2A3LYMU57JAGNYDIW5GI3TJYHQFEP4HO"/>
4+
<edm:WebResource rdf:about="https://heidicon.ub.uni-heidelberg.de/api/v1/objects/uuid/05b444c8-c47d-4b04-be2a-c86eaeffdb9e/file/id/221151/file_version/name/huge/disposition/inline">
5+
<dc:format>image/jpeg</dc:format>
6+
<edm:rights rdf:resource="http://rightsstatements.org/vocab/InC/1.0/"/>
7+
</edm:WebResource>
8+
<edm:TimeSpan rdf:about="#1910">
9+
<skos:prefLabel xml:lang="zxx">1910</skos:prefLabel>
10+
<dcterms:isPartOf rdf:resource="http://data.europeana.eu/timespan/20"></dcterms:isPartOf>
11+
<edm:begin>1910-01-01</edm:begin>
12+
<edm:end>1910-12-31</edm:end>
13+
<skos:notation rdf:datatype="http://id.loc.gov/datatypes/edtf/EDTF-level1">1910</skos:notation>
14+
</edm:TimeSpan>
15+
<edm:TimeSpan rdf:about="http://data.europeana.eu/timespan/20">
16+
<skos:prefLabel xml:lang="de">20. Jahrhundert</skos:prefLabel>
17+
<skos:prefLabel xml:lang="fi">1900-luku</skos:prefLabel>
18+
<skos:prefLabel xml:lang="ru">XX век</skos:prefLabel>
19+
<skos:prefLabel xml:lang="pt">Século XX</skos:prefLabel>
20+
<skos:prefLabel xml:lang="bg">20 век</skos:prefLabel>
21+
<skos:prefLabel xml:lang="lt">XX amžius</skos:prefLabel>
22+
<skos:prefLabel xml:lang="hr">20. stoljeće</skos:prefLabel>
23+
<skos:prefLabel xml:lang="lv">20. gadsimts</skos:prefLabel>
24+
<skos:prefLabel xml:lang="fr">XXe siècle</skos:prefLabel>
25+
<skos:prefLabel xml:lang="hu">20. század</skos:prefLabel>
26+
<skos:prefLabel xml:lang="sk">20. storočie</skos:prefLabel>
27+
<skos:prefLabel xml:lang="sl">20. stoletje</skos:prefLabel>
28+
<skos:prefLabel xml:lang="ga">20ú haois</skos:prefLabel>
29+
<skos:prefLabel xml:lang="ca">Segle XX</skos:prefLabel>
30+
<skos:prefLabel xml:lang="sv">1900-talet</skos:prefLabel>
31+
<skos:prefLabel xml:lang="el">20ός αιώνας</skos:prefLabel>
32+
<skos:prefLabel xml:lang="en">20th century</skos:prefLabel>
33+
<skos:prefLabel xml:lang="it">XX secolo</skos:prefLabel>
34+
<skos:prefLabel xml:lang="es">Siglo XX</skos:prefLabel>
35+
<skos:prefLabel xml:lang="et">20. sajand</skos:prefLabel>
36+
<skos:prefLabel xml:lang="cs">20. století</skos:prefLabel>
37+
<skos:prefLabel xml:lang="eu">XX. mendea</skos:prefLabel>
38+
<skos:prefLabel xml:lang="pl">XX wiek</skos:prefLabel>
39+
<skos:prefLabel xml:lang="da">20. århundrede</skos:prefLabel>
40+
<skos:prefLabel xml:lang="ro">Secolul al XX-lea</skos:prefLabel>
41+
<skos:prefLabel xml:lang="nl">20e eeuw</skos:prefLabel>
42+
<skos:altLabel xml:lang="sv">20:e århundradet</skos:altLabel>
43+
<skos:altLabel xml:lang="sv">20:e seklet</skos:altLabel>
44+
<skos:altLabel xml:lang="sv">1900-tal</skos:altLabel>
45+
<skos:altLabel xml:lang="sv">1900-talet (århundrade)</skos:altLabel>
46+
<skos:altLabel xml:lang="sv">1900-talet (sekel)</skos:altLabel>
47+
<skos:altLabel xml:lang="ru">20 век</skos:altLabel>
48+
<skos:altLabel xml:lang="pt">Século 20</skos:altLabel>
49+
<skos:altLabel xml:lang="pt">Século vinte</skos:altLabel>
50+
<skos:altLabel xml:lang="pt">Periodo 1901-2000</skos:altLabel>
51+
<skos:altLabel xml:lang="pt">Ciclo (1901-2000)</skos:altLabel>
52+
<skos:altLabel xml:lang="en">20th-century</skos:altLabel>
53+
<skos:altLabel xml:lang="en">20th-century</skos:altLabel>
54+
<skos:altLabel xml:lang="en">Twentieth century</skos:altLabel>
55+
<skos:altLabel xml:lang="en">The past century</skos:altLabel>
56+
<skos:altLabel xml:lang="en">History, 20th Century</skos:altLabel>
57+
<skos:altLabel xml:lang="en">XX Century</skos:altLabel>
58+
<skos:altLabel xml:lang="it">Novecento</skos:altLabel>
59+
<skos:altLabel xml:lang="it">20° secolo</skos:altLabel>
60+
<skos:altLabel xml:lang="it">'900</skos:altLabel>
61+
<skos:altLabel xml:lang="it">Novecento</skos:altLabel>
62+
<skos:altLabel xml:lang="fr">20e siècle</skos:altLabel>
63+
<skos:altLabel xml:lang="es">Siglo 20</skos:altLabel>
64+
<skos:altLabel xml:lang="es">El siglo pasado</skos:altLabel>
65+
<skos:altLabel xml:lang="es">Siglo veinte</skos:altLabel>
66+
<skos:altLabel xml:lang="es">Siglo XX después de Cristo</skos:altLabel>
67+
<skos:altLabel xml:lang="es">Siglo XX d. C.</skos:altLabel>
68+
<edm:begin>1901-01-01</edm:begin>
69+
<edm:end>2000-12-31</edm:end>
70+
<edm:isNextInSequence rdf:resource="http://data.europeana.eu/timespan/19"/>
71+
<owl:sameAs rdf:resource="http://www.wikidata.org/entity/Q6927"/>
72+
<owl:sameAs rdf:resource="http://id.loc.gov/authorities/names/sh2002012476"/>
73+
<owl:sameAs rdf:resource="http://id.loc.gov/authorities/names/sh85139020"/>
74+
<owl:sameAs rdf:resource="http://id.nlm.nih.gov/mesh/D049673"/>
75+
<owl:sameAs rdf:resource="https://www.freebase.com/m/089_x"/>
76+
<owl:sameAs rdf:resource="https://g.co/kg/m/089_x"/>
77+
<owl:sameAs rdf:resource="http://id.nlm.nih.gov/mesh/K01.400.504.968"/>
78+
<owl:sameAs rdf:resource="http://vocab.getty.edu/aat/300404514"/>
79+
<owl:sameAs rdf:resource="http://id.worldcat.org/fast/1159810"/>
80+
<owl:sameAs rdf:resource="http://dbpedia.org/resource/20th_century"/>
81+
</edm:TimeSpan>
82+
<skos:Concept rdf:about="https://d-nb.info/gnd/4029670-2">
83+
<skos:prefLabel>Karikatur</skos:prefLabel>
84+
</skos:Concept>
85+
<skos:Concept rdf:about="https://d-nb.info/gnd/4135144-7">
86+
<skos:prefLabel>Satirische Zeitschrift</skos:prefLabel>
87+
</skos:Concept>
88+
<skos:Concept rdf:about="https://d-nb.info/gnd/4021845-4">
89+
<skos:altLabel xml:lang="de">Graphik</skos:altLabel>
90+
<skos:closeMatch rdf:resource="http://id.loc.gov/authorities/subjects/sh85056474"/>
91+
<skos:closeMatch rdf:resource="https://data.bnf.fr/ark:/12148/cb11931640m"/>
92+
<skos:closeMatch rdf:resource="http://purl.org/bncf/tid/9850"/>
93+
<skos:closeMatch rdf:resource="https://datos.bne.es/resource/XX538203"/>
94+
<skos:exactMatch rdf:resource="http://www.wikidata.org/entity/Q1027879"/>
95+
<skos:note xml:lang="de">Formschlagwort für vorliegende thematisch erschließbare Graphik. Für einzelne graphische Blätter und Folgen von Graphikblättern - sowohl für Original- wie für Reproduktionsgraphik.</skos:note>
96+
<skos:prefLabel xml:lang="de">Grafik</skos:prefLabel>
97+
</skos:Concept>
98+
<ore:Aggregation rdf:about="/aggregation/provider/one-recor/item_2A3LYMU57JAGNYDIW5GI3TJYHQFEP4HO">
99+
<edm:aggregatedCHO rdf:resource="/one-recor/item_2A3LYMU57JAGNYDIW5GI3TJYHQFEP4HO"/>
100+
<edm:dataProvider>Ruprecht-Karls-Universit�t Heidelberg. Universit�tsbibliothek</edm:dataProvider>
101+
<edm:isShownAt rdf:resource="https://heidicon.ub.uni-heidelberg.de/detail/750097"/>
102+
<edm:isShownBy rdf:resource="https://heidicon.ub.uni-heidelberg.de/api/v1/objects/uuid/05b444c8-c47d-4b04-be2a-c86eaeffdb9e/file/id/221151/file_version/name/huge/disposition/inline"/>
103+
<edm:object rdf:resource="https://heidicon.ub.uni-heidelberg.de/api/v1/objects/uuid/05b444c8-c47d-4b04-be2a-c86eaeffdb9e/file/id/221151/file_version/name/huge/disposition/inline"/>
104+
<edm:provider rdf:resource="http://data.europeana.eu/organization/1548"></edm:provider>
105+
<edm:rights rdf:resource="http://rightsstatements.org/vocab/InC/1.0/"/>
106+
</ore:Aggregation>
107+
<ore:Proxy rdf:about="/proxy/provider/one-recor/item_2A3LYMU57JAGNYDIW5GI3TJYHQFEP4HO">
108+
<dc:date>1910 (Herstellung)</dc:date>
109+
<dc:identifier>G 5442-2 Folio RES (Inventarnummer)</dc:identifier>
110+
<dc:relation xml:lang="de">Verweis: http://digi.ub.uni-heidelberg.de/diglit/fb134/0250</dc:relation>
111+
<dc:subject rdf:resource="https://d-nb.info/gnd/4029670-2"></dc:subject>
112+
<dc:subject rdf:resource="https://d-nb.info/gnd/4135144-7"></dc:subject>
113+
<dc:title xml:lang="de">"Auf der Hofjagd"</dc:title>
114+
<dc:type rdf:resource="https://d-nb.info/gnd/4021845-4"></dc:type>
115+
<dcterms:alternative xml:lang="de">Fliegende Bl�tter (Serientitel)</dcterms:alternative>
116+
<dcterms:provenance xml:lang="de">Universit�tsbibliothek Heidelberg</dcterms:provenance>
117+
<dcterms:spatial xml:lang="de">M�nchen (Herstellung)</dcterms:spatial>
118+
<edm:europeanaProxy>false</edm:europeanaProxy>
119+
<ore:proxyFor rdf:resource="/one-recor/item_2A3LYMU57JAGNYDIW5GI3TJYHQFEP4HO"/>
120+
<ore:proxyIn rdf:resource="/aggregation/provider/one-recor/item_2A3LYMU57JAGNYDIW5GI3TJYHQFEP4HO"/>
121+
<edm:type>IMAGE</edm:type>
122+
</ore:Proxy>
123+
<ore:Proxy rdf:about="/proxy/europeana/one-recor/item_2A3LYMU57JAGNYDIW5GI3TJYHQFEP4HO">
124+
<dc:date rdf:resource="#1910"></dc:date>
125+
<dc:identifier>http://www.deutsche-digitale-bibliothek.de/item/2A3LYMU57JAGNYDIW5GI3TJYHQFEP4HO</dc:identifier>
126+
<edm:europeanaProxy>true</edm:europeanaProxy>
127+
<ore:proxyFor rdf:resource="/one-recor/item_2A3LYMU57JAGNYDIW5GI3TJYHQFEP4HO"/>
128+
<ore:proxyIn rdf:resource="/aggregation/europeana/one-recor/item_2A3LYMU57JAGNYDIW5GI3TJYHQFEP4HO"/>
129+
<ore:lineage rdf:resource="/proxy/provider/one-recor/item_2A3LYMU57JAGNYDIW5GI3TJYHQFEP4HO"/>
130+
</ore:Proxy>
131+
<edm:EuropeanaAggregation rdf:about="/aggregation/europeana/one-recor/item_2A3LYMU57JAGNYDIW5GI3TJYHQFEP4HO">
132+
<edm:aggregatedCHO rdf:resource="/one-recor/item_2A3LYMU57JAGNYDIW5GI3TJYHQFEP4HO"/>
133+
<edm:dataProvider xml:lang="en">Europeana Foundation</edm:dataProvider>
134+
<edm:provider xml:lang="en">Europeana Foundation</edm:provider>
135+
<edm:datasetName>idA_metisDatasetNameA</edm:datasetName>
136+
<edm:country>Greece</edm:country>
137+
<edm:language>el</edm:language>
138+
<edm:completeness>5</edm:completeness>
139+
</edm:EuropeanaAggregation>
140+
<foaf:Organization rdf:about="http://data.europeana.eu/organization/1548">
141+
<skos:prefLabel xml:lang="de">Deutsche Digitale Bibliothek</skos:prefLabel>
142+
<skos:prefLabel xml:lang="en">German Digital Library</skos:prefLabel>
143+
</foaf:Organization>
144+
</rdf:RDF>

0 commit comments

Comments
 (0)