diff --git a/src/main/java/de/mediathekview/mlib/filmlisten/FilmlistManager.java b/src/main/java/de/mediathekview/mlib/filmlisten/FilmlistManager.java index a717e372..d97e1660 100644 --- a/src/main/java/de/mediathekview/mlib/filmlisten/FilmlistManager.java +++ b/src/main/java/de/mediathekview/mlib/filmlisten/FilmlistManager.java @@ -34,13 +34,10 @@ public class FilmlistManager extends MessageCreator { private static final Logger LOG = LogManager.getLogger(FilmlistManager.class); private static final String TEMP_ENDING = "_TEMP"; private static FilmlistManager instance; - private final FilmlistOldFormatWriter filmlistOldFormatWriter; - private final FilmlistWriter filmlistWriter; + private FilmlistManager() { super(); - filmlistOldFormatWriter = new FilmlistOldFormatWriter(); - filmlistWriter = new FilmlistWriter(); } public static FilmlistManager getInstance() { @@ -107,11 +104,13 @@ public boolean save( final FilmlistFormats aFormat, final Filmlist aFilmlist, final Path aSavePath) { try { publishMessage(LibMessages.FILMLIST_WRITE_STARTED, aSavePath); - filmlistWriter.addAllMessageListener(messageListeners); - filmlistOldFormatWriter.addAllMessageListener(messageListeners); if (aFormat.isOldFormat()) { + final FilmlistOldFormatWriter filmlistOldFormatWriter = new FilmlistOldFormatWriter(); + filmlistOldFormatWriter.addAllMessageListener(messageListeners); return save(filmlistOldFormatWriter, aFormat, aFilmlist, aSavePath); } else { + final FilmlistWriter filmlistWriter = new FilmlistWriter(); + filmlistWriter.addAllMessageListener(messageListeners); return save(filmlistWriter, aFormat, aFilmlist, aSavePath); } } finally { diff --git a/src/main/java/de/mediathekview/mlib/filmlisten/reader/FilmlistOldFormatReader.java b/src/main/java/de/mediathekview/mlib/filmlisten/reader/FilmlistOldFormatReader.java index 155efcf9..d24d824d 100644 --- a/src/main/java/de/mediathekview/mlib/filmlisten/reader/FilmlistOldFormatReader.java +++ b/src/main/java/de/mediathekview/mlib/filmlisten/reader/FilmlistOldFormatReader.java @@ -51,8 +51,10 @@ public Optional read(InputStream aInputStream) { long start = System.currentTimeMillis(); Filmlist filmlist = new Filmlist(); debug = "LINE " + cnt; - // - + // these must be reset for each loading run + this.sender = ""; + this.thema = ""; + this.cnt = 0; try (JsonReader jsonReader = new JsonReader(new InputStreamReader(aInputStream, StandardCharsets.UTF_8))) { headerMeta(jsonReader, filmlist); @@ -67,7 +69,6 @@ public Optional read(InputStream aInputStream) { throw(e); } } - cnt++; } } catch (IOException e) { LOG.error(e); @@ -248,25 +249,25 @@ private Optional rebuildUrl(URL urlNromal, String targetUrl) { //////////////////////////////////////////////////////////// - protected String readRecord01Sender(String in, String sender) { + private String readRecord01Sender(String in, String sender) { if (!in.isBlank()) { sender = in; } return sender; } - protected String readRecord02Thema(String in, String thema) { + private String readRecord02Thema(String in, String thema) { if (!in.isBlank()) { thema = in; } return thema; } - protected String readRecord03Titel(String in) { + private String readRecord03Titel(String in) { return in; } - protected LocalDate readRecord04Datum(String in) { + private LocalDate readRecord04Datum(String in) { if (StringUtils.isNotBlank(in)) { try { return LocalDate.parse(in, DATE_FORMATTER); @@ -277,7 +278,7 @@ protected LocalDate readRecord04Datum(String in) { return DEFAULT_DATE; } - protected LocalTime readRecord05Zeit(String in) { + private LocalTime readRecord05Zeit(String in) { if (StringUtils.isNotBlank(in)) { try { return LocalTime.parse(in, TIME_FORMATTER); @@ -288,7 +289,7 @@ protected LocalTime readRecord05Zeit(String in) { return LocalTime.MIDNIGHT; } - protected Duration readRecord06Dauer(String in) { + private Duration readRecord06Dauer(String in) { if (StringUtils.isNotBlank(in)) { try { return Duration.between(LocalTime.MIDNIGHT, LocalTime.parse(in)); @@ -299,7 +300,7 @@ protected Duration readRecord06Dauer(String in) { return Duration.ZERO; } - protected long readRecord07Groesse(String in) { + private long readRecord07Groesse(String in) { if (StringUtils.isNotBlank(in)) { try { return Long.parseLong(in)*1024; // oldFilmlist format is MB - new DM is KB @@ -310,11 +311,11 @@ protected long readRecord07Groesse(String in) { return 0L; } - protected String readRecord08Beschreibung(String in) { + private String readRecord08Beschreibung(String in) { return in; } - protected URL readRecord09Url(String in) { + private URL readRecord09Url(String in) { if (!in.isBlank()) { try { return new URL(in); @@ -325,7 +326,7 @@ protected URL readRecord09Url(String in) { return null; } - protected URL readRecord10Website(String in) { + private URL readRecord10Website(String in) { if (!in.isBlank() && in.startsWith("http")) { try { return new URL(in); @@ -336,7 +337,7 @@ protected URL readRecord10Website(String in) { return null; } - protected URL readRecord11Untertitel(String in) { + private URL readRecord11Untertitel(String in) { if (!in.isBlank() && in.startsWith("http")) { try { return new URL(in); @@ -347,35 +348,35 @@ protected URL readRecord11Untertitel(String in) { return null; } - protected String readRecord12UrlRTMP(String in) { + private String readRecord12UrlRTMP(String in) { return in; } - protected String readRecord13UrlKlein(String in) { + private String readRecord13UrlKlein(String in) { return in; } - protected String readRecord14UrlRTMPKlein(String in) { + private String readRecord14UrlRTMPKlein(String in) { return in; } - - protected String readRecord15UrlHD(String in) { + + private String readRecord15UrlHD(String in) { return in; } - protected String readRecord16UrlRTMPHD(String in) { + private String readRecord16UrlRTMPHD(String in) { return in; } - protected String readRecord17DatumL(String in) { + private String readRecord17DatumL(String in) { return in; } - protected String readRecord18UrlHistory(String in) { + private String readRecord18UrlHistory(String in) { return in; } - protected Collection readRecord19Geo(String in) { + private Collection readRecord19Geo(String in) { final Collection geoLocations = new ArrayList<>(); final GeoLocations singleGeoLocation = GeoLocations.getFromDescription(in); @@ -393,7 +394,7 @@ protected Collection readRecord19Geo(String in) { return geoLocations; } - protected Boolean readRecord20Neu(String in) { + private Boolean readRecord20Neu(String in) { return Boolean.parseBoolean(in); } diff --git a/src/main/java/de/mediathekview/mlib/filmlisten/writer/FilmlistOldFormatWriter.java b/src/main/java/de/mediathekview/mlib/filmlisten/writer/FilmlistOldFormatWriter.java index c17d2a46..edaefad9 100644 --- a/src/main/java/de/mediathekview/mlib/filmlisten/writer/FilmlistOldFormatWriter.java +++ b/src/main/java/de/mediathekview/mlib/filmlisten/writer/FilmlistOldFormatWriter.java @@ -39,13 +39,17 @@ public class FilmlistOldFormatWriter extends AbstractFilmlistWriter { private static final char GEO_SPLITTERATOR = '-'; private static final DateTimeFormatter DATE_TIME_FORMAT = DateTimeFormatter.ofLocalizedDateTime(MEDIUM, SHORT).withLocale(Locale.GERMANY); - protected String sender = ""; - protected String thema = ""; - protected int cnt; + private String sender = ""; + private String thema = ""; + private int cnt = 0; @Override public boolean write(Filmlist filmlist, OutputStream outputStream) throws IOException { long start = System.currentTimeMillis(); + // these must be reset otherwise we may have old values in here + this.sender = ""; + this.thema = ""; + this.cnt = 0; try { LOG.info("start writting data"); JsonWriter jsonWriter = new JsonWriter(new OutputStreamWriter(outputStream,StandardCharsets.UTF_8)); @@ -70,7 +74,7 @@ public boolean write(Filmlist filmlist, OutputStream outputStream) throws IOExce return true; } - protected void writeMetaHeader(Filmlist list, JsonWriter jsonWriter ) throws IOException { + private void writeMetaHeader(Filmlist list, JsonWriter jsonWriter ) throws IOException { jsonWriter.name("Filmliste").beginArray(); jsonWriter.value(writeMetaHeader01CreationDate(list)); jsonWriter.value(writeMetaHeader02CreationDateUTC(list)); @@ -81,28 +85,28 @@ protected void writeMetaHeader(Filmlist list, JsonWriter jsonWriter ) throws IOE } - protected String writeMetaHeader01CreationDate(Filmlist in) { + private String writeMetaHeader01CreationDate(Filmlist in) { return DATE_TIME_FORMAT.format(in.getCreationDate()); } - protected String writeMetaHeader02CreationDateUTC(Filmlist in) { + private String writeMetaHeader02CreationDateUTC(Filmlist in) { return DATE_TIME_FORMAT.format(in.getCreationDate().atZone(ZoneOffset.UTC)); } - protected String writeMetaHeader03Version(Filmlist in) { + private String writeMetaHeader03Version(Filmlist in) { return "4"; } - protected String writeMetaHeader04VErsionLong(Filmlist in) { + private String writeMetaHeader04VErsionLong(Filmlist in) { return "MSearch [Vers.: 4.0.1]"; } - protected String writeMetaHeader05Id(Filmlist in) { + private String writeMetaHeader05Id(Filmlist in) { return in.getListId().toString(); } - protected void writeColumnHeader(JsonWriter jsonWriter) throws IOException { + private void writeColumnHeader(JsonWriter jsonWriter) throws IOException { jsonWriter.name("Filmliste").beginArray(); jsonWriter.value("Sender"); jsonWriter.value("Thema"); @@ -127,7 +131,7 @@ protected void writeColumnHeader(JsonWriter jsonWriter) throws IOException { jsonWriter.endArray(); } - protected void writeRecord(AbstractMediaResource film, JsonWriter jsonWriter) throws IOException { + private void writeRecord(AbstractMediaResource film, JsonWriter jsonWriter) throws IOException { jsonWriter.name("X").beginArray(); jsonWriter.value(writeRecord01Sender(film, this.sender)); jsonWriter.value(writeRecord02Thema(film, this.thema)); @@ -152,7 +156,7 @@ protected void writeRecord(AbstractMediaResource film, JsonWriter jsonWriter) jsonWriter.endArray(); } - protected String writeRecord01Sender(AbstractMediaResource in, String aSender) { + private String writeRecord01Sender(AbstractMediaResource in, String aSender) { if (!aSender.equalsIgnoreCase(in.getSenderName())) { this.sender = in.getSenderName(); return in.getSenderName(); @@ -161,7 +165,7 @@ protected String writeRecord01Sender(AbstractMediaResource in, String aSender } } - protected String writeRecord02Thema(AbstractMediaResource in, String aThema) { + private String writeRecord02Thema(AbstractMediaResource in, String aThema) { if(!aThema.equalsIgnoreCase(in.getThema()) ) { this.thema = in.getThema(); return in.getThema(); @@ -170,102 +174,102 @@ protected String writeRecord02Thema(AbstractMediaResource in, String aThema) } } - protected String writeRecord03Titel(AbstractMediaResource in) { + private String writeRecord03Titel(AbstractMediaResource in) { return in.getTitel(); } - protected String writeRecord04Datum(AbstractMediaResource in) { + private String writeRecord04Datum(AbstractMediaResource in) { return in.getTime().format(DATE_FORMATTER); } - protected String writeRecord05Zeit(AbstractMediaResource in) { + private String writeRecord05Zeit(AbstractMediaResource in) { return in.getTime().format(TIME_FORMATTER); } - protected String writeRecord06Dauer(AbstractMediaResource in) { + private String writeRecord06Dauer(AbstractMediaResource in) { if (!(in instanceof Podcast pIn) || pIn.getDuration().isZero()) { return ""; } return LocalTime.MIDNIGHT.plus(pIn.getDuration()).format(DateTimeFormatter.ofPattern(DURATION_FORMAT)); } - protected String writeRecord07Groesse(AbstractMediaResource in) { + private String writeRecord07Groesse(AbstractMediaResource in) { if ((in instanceof Podcast pIn) && pIn.getUrl(Resolution.NORMAL) != null) return (pIn.getUrl(Resolution.NORMAL).getFileSize()/1024) + ""; return ""; } - protected String writeRecord08Beschreibung(AbstractMediaResource in) { + private String writeRecord08Beschreibung(AbstractMediaResource in) { return in.getBeschreibung(); } - protected String writeRecord09UrlNormal(AbstractMediaResource in) { + private String writeRecord09UrlNormal(AbstractMediaResource in) { if ((in instanceof Podcast pIn) && pIn.getUrl(Resolution.NORMAL) != null) return pIn.getUrl(Resolution.NORMAL).getUrl().toString(); return ""; } - protected String writeRecord10Website(AbstractMediaResource in) { + private String writeRecord10Website(AbstractMediaResource in) { if (in.getWebsite().isPresent()) { return in.getWebsite().get().toString(); } return ""; } - protected String writeRecord11Untertitel(AbstractMediaResource in) { + private String writeRecord11Untertitel(AbstractMediaResource in) { if ((in instanceof Film fIn) && !fIn.getSubtitles().isEmpty()) { return fIn.getSubtitles().toArray()[0].toString(); } return ""; } - protected String writeRecord12UrlRTMP(AbstractMediaResource in) { + private String writeRecord12UrlRTMP(AbstractMediaResource in) { return ""; } - protected String writeRecord13UrlKlein(AbstractMediaResource in) { + private String writeRecord13UrlKlein(AbstractMediaResource in) { if ((in instanceof Podcast pIn) && in.getUrl(Resolution.SMALL) != null && pIn.getUrl(Resolution.NORMAL) != null) { return reduceUrl(pIn.getUrl(Resolution.NORMAL).getUrl().toString(), pIn.getUrl(Resolution.SMALL).getUrl().toString()); } return ""; } - protected String writeRecord14UrlKleinRTMP(AbstractMediaResource in) { + private String writeRecord14UrlKleinRTMP(AbstractMediaResource in) { return ""; } - protected String writeRecord15UrlHD(AbstractMediaResource in) { + private String writeRecord15UrlHD(AbstractMediaResource in) { if ((in instanceof Podcast pIn) && in.getUrl(Resolution.HD) != null && in.getUrl(Resolution.NORMAL) != null) { return reduceUrl(pIn.getUrl(Resolution.NORMAL).getUrl().toString(), pIn.getUrl(Resolution.HD).getUrl().toString()); } return ""; } - protected String writeRecord16UrlHdRTMP(AbstractMediaResource in) { + private String writeRecord16UrlHdRTMP(AbstractMediaResource in) { return ""; } - protected String writeRecord17DatumL(AbstractMediaResource in) { + private String writeRecord17DatumL(AbstractMediaResource in) { final ZonedDateTime zonedDateTime = in.getTime().atZone(ZONE_ID); return zonedDateTime.toEpochSecond()+""; } - protected String writeRecord18UrlHistory(AbstractMediaResource in) { + private String writeRecord18UrlHistory(AbstractMediaResource in) { return ""; } - protected String writeRecord19Geo(AbstractMediaResource in) { + private String writeRecord19Geo(AbstractMediaResource in) { return geolocationsToStirng(in.getGeoLocations()); } - protected String writeRecord20Neu(AbstractMediaResource in) { + private String writeRecord20Neu(AbstractMediaResource in) { if ((in instanceof Podcast pIn)) { return Boolean.toString(pIn.isNeu()); } return Boolean.toString(false); } - protected String reduceUrl(final String aBaseUrl, final String aUrlToReduce) { + private String reduceUrl(final String aBaseUrl, final String aUrlToReduce) { final StringBuilder urlIntersectionBuilder = new StringBuilder(); for (int i = 0; i < aBaseUrl.length() diff --git a/src/main/java/de/mediathekview/mlib/tool/FileSizeDeterminer.java b/src/main/java/de/mediathekview/mlib/tool/FileSizeDeterminer.java index 50ebbdbf..28743906 100644 --- a/src/main/java/de/mediathekview/mlib/tool/FileSizeDeterminer.java +++ b/src/main/java/de/mediathekview/mlib/tool/FileSizeDeterminer.java @@ -8,97 +8,42 @@ import org.jetbrains.annotations.NotNull; import java.io.IOException; -import java.util.Optional; import static jakarta.ws.rs.core.HttpHeaders.CONTENT_LENGTH; import static jakarta.ws.rs.core.HttpHeaders.CONTENT_TYPE; public class FileSizeDeterminer { private static final Logger LOG = LogManager.getLogger(FileSizeDeterminer.class); - private static final int BYTE_TO_MIB = 1024; - private static final int BYTE_TO_MB = 1000; - private static final String PROTOCOL_RTMP = "rtmp"; - private static final String FILE_TYPE_M3U8 = "m3u8"; - private final String url; private final OkHttpClient client; // - private Optional fileSizeInByte = Optional.empty(); - private Optional responsePath = Optional.empty(); - private Optional responseContentType = Optional.empty(); - - - /* - * get the file size of the url in byte - */ - public long getFileSizeInByte() { - if (fileSizeInByte.isEmpty()) { - getFileSizeForBuilder(); - } - return fileSizeInByte.orElse(-1L); - } - - /* - * get the path of the response which may differ to request url for redirects - */ - public String getResponsePath() { - if (responsePath.isEmpty()) { - getFileSizeInByte(); - } - return responsePath.orElse(""); - } - - /* - * get the content type of the reponse message - */ - - public String getResponseContentType() { - if (responseContentType.isEmpty()) { - getFileSizeInByte(); - } - return responseContentType.orElse(""); - } - - + - /** - * Builds the determiner with the default read- and connect timeout of 30 seconds. - * - * @param aUrl The url of the file. - */ - public FileSizeDeterminer(final String aUrl) { - this(aUrl, 30L, 30L); + public FileSizeDeterminer() { + this(30L,30L); } - /** - * @param aUrl The url of the file. - * @param connectTimeoutInSeconds The connection timeout in seconds. - * @param readTimeoutInSeconds The read timeout in seconds. - */ public FileSizeDeterminer( - final String aUrl, final long connectTimeoutInSeconds, final long readTimeoutInSeconds) { - url = aUrl; + final long connectTimeoutInSeconds, final long readTimeoutInSeconds) { client = new OkHttpClientBuilder() .withConnectTimeout(connectTimeoutInSeconds) .withReadTimeout(readTimeoutInSeconds) .build(); } - - /** @return The file size in bytes. */ - public Long getFileSizeForBuilder() { - return getFileSizeByRequest(RequestType.HEAD); + public RespoonseInfo getRequestInfo(final String url) { + return getRequestInfo(url, RequestType.HEAD); } - private Long getFileSizeByRequest(final RequestType requestType) { - // Cant determine the file size of rtmp and m3u8. - if (!url.startsWith(PROTOCOL_RTMP) && !url.endsWith(FILE_TYPE_M3U8)) { + public RespoonseInfo getRequestInfo(final String url, final RequestType requestType) { try (final Response response = - client.newCall(createRequestBuilderForRequestType(requestType).build()).execute()) { - final String contentLengthHeader = response.header(CONTENT_LENGTH); - responseContentType = Optional.of(response.header(CONTENT_TYPE, "")); - responsePath = Optional.of(response.request().url().encodedPath()); - fileSizeInByte = Optional.of(parseContentLength(contentLengthHeader)); - return fileSizeInByte.get(); + client.newCall(createRequestBuilderForRequestType(url, requestType).build()).execute()) { + RespoonseInfo respoonseInfo = new RespoonseInfo( + parseContentLength(response.header(CONTENT_LENGTH)), + response.code(), + response.header(CONTENT_TYPE, ""), + response.request().url().encodedPath() + ); + return respoonseInfo; } catch (final IOException ioException) { LOG.error( "Something went wrong determining the file size of \"{}\" with {} request.", @@ -106,15 +51,14 @@ private Long getFileSizeByRequest(final RequestType requestType) { requestType); if (requestType.equals(RequestType.HEAD)) { LOG.info("Retrying the file size determination with GET request."); - return getFileSizeByRequest(RequestType.GET); + return getRequestInfo(url, RequestType.GET); } } - } - return -1L; + return null; } @NotNull - private Request.Builder createRequestBuilderForRequestType(final RequestType requestType) { + private Request.Builder createRequestBuilderForRequestType(final String url, final RequestType requestType) { final Request.Builder requestBuilder; switch (requestType) { case GET: @@ -129,11 +73,11 @@ private Request.Builder createRequestBuilderForRequestType(final RequestType req return requestBuilder; } - /** @return The file size in MiB. */ - public Long getFileSizeInMiB() { - return getFileSizeForBuilder() / BYTE_TO_MIB; + private enum RequestType { + GET, + HEAD } - + @NotNull private Long parseContentLength(final String contentLengthHeader) { try { @@ -146,14 +90,33 @@ private Long parseContentLength(final String contentLengthHeader) { return -1L; } } - - /** @return The file size in MB. */ - public Long getFileSizeInMB() { - return getFileSizeForBuilder() / BYTE_TO_MB; + + public class RespoonseInfo { + private Long size; + private int code; + private String contentType; + private String path; + + public RespoonseInfo(Long size, int code, String contentType, String path) { + super(); + this.size = size; + this.code = code; + this.contentType = contentType; + this.path = path; + } + + public Long getSize() { + return size; + } + public int getCode() { + return code; + } + public String getContentType() { + return contentType; + } + public String getPath() { + return path; + } } - private enum RequestType { - GET, - HEAD - } } diff --git a/src/test/java/de/mediathekview/mlib/tool/FileSizeDeterminerTest.java b/src/test/java/de/mediathekview/mlib/tool/FileSizeDeterminerTest.java index 4c1d50f7..62fae6ff 100644 --- a/src/test/java/de/mediathekview/mlib/tool/FileSizeDeterminerTest.java +++ b/src/test/java/de/mediathekview/mlib/tool/FileSizeDeterminerTest.java @@ -9,6 +9,7 @@ import static com.github.tomakehurst.wiremock.client.WireMock.*; import static com.github.tomakehurst.wiremock.core.WireMockConfiguration.options; import static jakarta.ws.rs.core.HttpHeaders.CONTENT_LENGTH; +import static jakarta.ws.rs.core.HttpHeaders.CONTENT_TYPE; import static org.assertj.core.api.AssertionsForInterfaceTypes.assertThat; class FileSizeDeterminerTest { @@ -21,7 +22,7 @@ public static void setUpWiremock() { wireMockServer.stubFor( head(urlEqualTo("/" + TEST_FILE_NAME)) .willReturn( - aResponse().withStatus(200).withHeader(CONTENT_LENGTH, "5643"))); + aResponse().withStatus(200).withHeader(CONTENT_LENGTH, "5643").withHeader(CONTENT_TYPE, "text/html"))); } @BeforeEach @@ -36,20 +37,20 @@ public void stopWireMock() { @Test void testGetFileSize() { - assertThat(getClassUnderTest().getFileSizeForBuilder()).isEqualTo(5643L); + assertThat(getClassUnderTest().getRequestInfo(wireMockServer.baseUrl() + TEST_FILE_URL).getSize()).isEqualTo(5643L); } - + @Test - void testGetFileSizeMiB() { - assertThat(getClassUnderTest().getFileSizeInMiB()).isEqualTo(5L); + void testGetStatusCode() { + assertThat(getClassUnderTest().getRequestInfo(wireMockServer.baseUrl() + TEST_FILE_URL).getCode()).isEqualTo(200); } - + @Test - void testGetFileSizeMB() { - assertThat(getClassUnderTest().getFileSizeInMB()).isEqualTo(5L); + void testGetContentType() { + assertThat(getClassUnderTest().getRequestInfo(wireMockServer.baseUrl() + TEST_FILE_URL).getContentType()).isEqualTo("text/html"); } private FileSizeDeterminer getClassUnderTest() { - return new FileSizeDeterminer(wireMockServer.baseUrl() + TEST_FILE_URL); + return new FileSizeDeterminer(); } }