Merge pull request #154 from mediathekview/dev/OldFilmlistEntryToFilm

fix invalid urls, allow two different time formats, rework defaults

src/main/java/de/mediathekview/mlib/compression/CompressionManager.java

@@ -33,12 +33,12 @@ public OutputStream compress(
       final CompressionType aCompressionType, final OutputStream aOutputStream) throws IOException {
     switch (aCompressionType) {
       case XZ:
-        return new XZCompressorOutputStream(aOutputStream, 9);
+        return new XZCompressorOutputStream(new BufferedOutputStream(aOutputStream, 512000), 9);
       case GZIP:
-        return new MediathekviewGZIPOutputStrean(aOutputStream);
+        return new MediathekviewGZIPOutputStrean(new BufferedOutputStream(aOutputStream, 512000));
       case BZIP:
         // This uses already the best compression.
-        return new BZip2CompressorOutputStream(aOutputStream);
+        return new BZip2CompressorOutputStream(new BufferedOutputStream(aOutputStream, 512000));
       default:
         throw new IllegalArgumentException(
             String.format("The type \"%s\" isn't supported.", aCompressionType.name()));
@@ -75,7 +75,7 @@ public void compress(
             : aTargetPath.resolveSibling(
                 aTargetPath.getFileName().toString() + aCompressionType.getFileEnding());
 
-    try (final InputStream input = new BufferedInputStream(Files.newInputStream(aSourceFile));
+    try (final BufferedInputStream input = new BufferedInputStream(Files.newInputStream(aSourceFile), 512000);
          final OutputStream output = compress(aCompressionType, Files.newOutputStream(targetPath))) {
       fastChannelCopy(Channels.newChannel(input), Channels.newChannel(output));
     }

src/main/java/de/mediathekview/mlib/daten/AbstractMediaResource.java

@@ -32,7 +32,7 @@ public abstract class AbstractMediaResource<T extends Serializable> implements S
     beschreibung = "";
   }
 
-  public AbstractMediaResource(
+  protected AbstractMediaResource(
       final UUID aUuid,
       final Sender aSender,
       final String aTitel,
@@ -53,7 +53,7 @@ public AbstractMediaResource(
     beschreibung = "";
   }
 
-  public AbstractMediaResource(final AbstractMediaResource<T> copyObj) {
+  protected AbstractMediaResource(final AbstractMediaResource<T> copyObj) {
     super();
     uuid = copyObj.uuid;
     geoLocations = copyObj.geoLocations;
@@ -110,7 +110,7 @@ public boolean equals(final Object obj) {
     if (getClass() != obj.getClass()) {
       return false;
     }
-    final AbstractMediaResource other = (AbstractMediaResource) obj;
+    final AbstractMediaResource<T> other = (AbstractMediaResource<T>) obj;
     if (sender != other.sender) {
       return false;
     }

src/main/java/de/mediathekview/mlib/filmlisten/FilmToFakeJsonConverter.java

@@ -5,6 +5,8 @@
 import org.apache.logging.log4j.LogManager;
 import org.apache.logging.log4j.Logger;
 
+import java.io.IOException;
+import java.io.OutputStreamWriter;
 import java.io.Serializable;
 import java.net.URL;
 import java.time.*;
@@ -46,11 +48,11 @@ public class FilmToFakeJsonConverter {
         "neu"
       };
   private static final String OUTPUT_PATTERN =
-      "\"X\": [\"%s\",%s,%s,\"%s\",\"%s\",\"%s\",\"%s\",%s,\"%s\",\"%s\",\"%s\",\"%s\",\"%s\",\"%s\",\"%s\",\"%s\",\"%s\",\"%s\",\"%s\",\"%s\"]";
+      "\"X\":[\"%s\",%s,%s,\"%s\",\"%s\",\"%s\",\"%s\",%s,\"%s\",\"%s\",\"%s\",\"%s\",\"%s\",\"%s\",\"%s\",\"%s\",\"%s\",\"%s\",\"%s\",\"%s\"]";
   private static final String META_INFORMATION_PATTERN =
-      "\"Filmliste\": [\"%s\",\"%s\",\"%s\",\"%s\",\"%s\"]";
+      "\"Filmliste\":[\"%s\",\"%s\",\"%s\",\"%s\",\"%s\"]";
   private static final String COLUMNNAMES_PATTERN =
-      "\"Filmliste\": [\"%s\",\"%s\",\"%s\",\"%s\",\"%s\",\"%s\",\"%s\",\"%s\",\"%s\",\"%s\",\"%s\",\"%s\",\"%s\",\"%s\",\"%s\",\"%s\",\"%s\",\"%s\",\"%s\",\"%s\"]";
+      "\"Filmliste\":[\"%s\",\"%s\",\"%s\",\"%s\",\"%s\",\"%s\",\"%s\",\"%s\",\"%s\",\"%s\",\"%s\",\"%s\",\"%s\",\"%s\",\"%s\",\"%s\",\"%s\",\"%s\",\"%s\",\"%s\"]";
   private static final char SPLITTERATOR = ',';
   private static final char FAKE_JSON_BEGIN = '{';
   private static final char FAKE_JSON_END = '}';
@@ -65,37 +67,35 @@ public class FilmToFakeJsonConverter {
   private String lastSender;
   private String lastThema;
 
-  public String toFakeJson(
+  public void toFakeJson(
       final List<AbstractMediaResource<?>> aResources,
+      final OutputStreamWriter outputstream,
       final String aFilmlisteDatum,
       final String aFilmlisteDatumGmt,
       final String aFilmlisteVersion,
       final String aFilmlisteProgramm,
-      final String aFilmlisteId) {
-    final StringBuilder fakeJsonBuilder = new StringBuilder();
-    fakeJsonBuilder.append(FAKE_JSON_BEGIN);
-    fakeJsonBuilder.append(System.lineSeparator());
-
-    fakeJsonBuilder.append(
+      final String aFilmlisteId) throws IOException {
+    outputstream.write(FAKE_JSON_BEGIN);
+    outputstream.write(
         String.format(
             META_INFORMATION_PATTERN,
             aFilmlisteDatum,
             aFilmlisteDatumGmt,
             aFilmlisteVersion,
             aFilmlisteProgramm,
             aFilmlisteId));
-    appendEnd(fakeJsonBuilder, false);
+    appendEnd(outputstream, false);
 
-    fakeJsonBuilder.append(String.format(COLUMNNAMES_PATTERN, COLUMNNAMES));
-    appendEnd(fakeJsonBuilder, false);
+    outputstream.write(String.format(COLUMNNAMES_PATTERN, COLUMNNAMES));
+    appendEnd(outputstream, false);
 
     lastSender = "";
     lastThema = "";
     for (final AbstractMediaResource<?> mediaResource :
         aResources.stream().filter(Objects::nonNull).toList()) {
       try {
         resourceToFakeJson(
-            fakeJsonBuilder,
+            outputstream,
             mediaResource,
             mediaResource.equals(aResources.get(aResources.size() - 1)));
       } catch (final Exception exception) {
@@ -104,15 +104,14 @@ public String toFakeJson(
       }
     }
 
-    fakeJsonBuilder.append(FAKE_JSON_END);
-    return fakeJsonBuilder.toString();
+    outputstream.write(FAKE_JSON_END);
+
   }
 
-  private void appendEnd(final StringBuilder fakeJsonBuilder, final boolean aIsLastFilm) {
+  private void appendEnd(final OutputStreamWriter outputstream, final boolean aIsLastFilm) throws IOException {
     if (!aIsLastFilm) {
-      fakeJsonBuilder.append(SPLITTERATOR);
+      outputstream.write(SPLITTERATOR);
     }
-    fakeJsonBuilder.append(System.lineSeparator());
   }
 
   private String durationToString(final Duration aDuration) {
@@ -201,17 +200,17 @@ private long convertDateTimeToLong(final LocalDateTime dateTime) {
   }
 
   private void resourceToFakeJson(
-      final StringBuilder fakeJsonBuilder,
+      final OutputStreamWriter outputstream,
       final AbstractMediaResource<?> aMediaResource,
-      final boolean aIsLastFilm) {
+      final boolean aIsLastFilm) throws IOException {
 
     final String title = aMediaResource.getTitel();
-    appendDefaultEntry(aMediaResource, title, fakeJsonBuilder, aIsLastFilm);
-    appendAudioDescriptionEntry(aMediaResource, title, fakeJsonBuilder, aIsLastFilm);
-    appendSignLanguageEntry(aMediaResource, title, fakeJsonBuilder, aIsLastFilm);
+    appendDefaultEntry(aMediaResource, title, outputstream, aIsLastFilm);
+    appendAudioDescriptionEntry(aMediaResource, title, outputstream, aIsLastFilm);
+    appendSignLanguageEntry(aMediaResource, title, outputstream, aIsLastFilm);
   }
 
-  private void appendDefaultEntry(AbstractMediaResource<?> aMediaResource, String title, StringBuilder fakeJsonBuilder, boolean aIsLastFilm) {
+  private void appendDefaultEntry(AbstractMediaResource<?> aMediaResource, String title, OutputStreamWriter outputstream, boolean aIsLastFilm) throws IOException {
     final String url;
     String urlKlein = "";
     String urlHd = "";
@@ -228,15 +227,15 @@ private void appendDefaultEntry(AbstractMediaResource<?> aMediaResource, String
       }
 
       appendMediaResource(
-          fakeJsonBuilder, aMediaResource, title, urlKlein, url, urlHd, aIsLastFilm);
+          outputstream, aMediaResource, title, urlKlein, url, urlHd, aIsLastFilm);
     }
   }
 
   private void appendAudioDescriptionEntry(
       final AbstractMediaResource<?> aMediaResource,
       final String aTitle,
-      final StringBuilder fakeJsonBuilder,
-      final boolean aIsLastFilm) {
+      final OutputStreamWriter outputstream,
+      final boolean aIsLastFilm) throws IOException {
     final String url;
     String urlSmall = "";
     String urlHd = "";
@@ -258,7 +257,7 @@ private void appendAudioDescriptionEntry(
 
         final String titleAudioDescription = aTitle + " (Audiodeskription)";
         appendMediaResource(
-            fakeJsonBuilder,
+            outputstream,
             aMediaResource,
             titleAudioDescription,
             urlSmall,
@@ -272,8 +271,8 @@ private void appendAudioDescriptionEntry(
   private void appendSignLanguageEntry(
       final AbstractMediaResource<?> aMediaResource,
       final String aTitle,
-      final StringBuilder fakeJsonBuilder,
-      final boolean aIsLastFilm) {
+      final OutputStreamWriter outputstream,
+      final boolean aIsLastFilm) throws IOException {
     final String url;
     String urlSmall = "";
     String urlHd = "";
@@ -295,19 +294,19 @@ private void appendSignLanguageEntry(
 
         final String titleSignLanguage = aTitle + " (Gebärdensprache)";
         appendMediaResource(
-            fakeJsonBuilder, aMediaResource, titleSignLanguage, urlSmall, url, urlHd, aIsLastFilm);
+            outputstream, aMediaResource, titleSignLanguage, urlSmall, url, urlHd, aIsLastFilm);
       }
     }
   }
 
   private void appendMediaResource(
-      final StringBuilder fakeJsonBuilder,
+      final OutputStreamWriter outputstream,
       final AbstractMediaResource<?> aMediaResource,
       final String aTitle,
       String aUrlSmall,
       final String aUrlNormal,
       String aUrlHd,
-      final boolean aIsLastFilm) {
+      final boolean aIsLastFilm) throws IOException {
     final Gson gson = new Gson();
 
     final String thema = setThema(aMediaResource);
@@ -318,7 +317,7 @@ private void appendMediaResource(
 
     final String website = aMediaResource.getWebsite().map(URL::toString).orElse("");
 
-    fakeJsonBuilder.append(
+    outputstream.write(
         String.format(
             OUTPUT_PATTERN,
             sender,
@@ -348,7 +347,7 @@ private void appendMediaResource(
             geolocationsToStirng(aMediaResource.getGeoLocations()),
             aMediaResource instanceof Podcast podcast && podcast.isNeu()));
 
-    appendEnd(fakeJsonBuilder, aIsLastFilm);
+    appendEnd(outputstream, aIsLastFilm);
   }
 
   private String setSender(final AbstractMediaResource<?> aMediaResource) {

src/main/java/de/mediathekview/mlib/filmlisten/FilmlistManager.java

@@ -18,6 +18,7 @@
 import org.apache.logging.log4j.LogManager;
 import org.apache.logging.log4j.Logger;
 
+import java.io.BufferedInputStream;
 import java.io.BufferedWriter;
 import java.io.IOException;
 import java.io.InputStream;
@@ -79,8 +80,9 @@ private InputStream decompressInputStreamIfFormatNeedsTo(
 
   public Optional<Filmlist> importList(final FilmlistFormats aFormat, final Path aFilePath)
       throws IOException {
-    try (final InputStream fileInputStream = Files.newInputStream(aFilePath)) {
-      return importList(aFormat, fileInputStream);
+    try (final InputStream fileInputStream = Files.newInputStream(aFilePath);
+        final BufferedInputStream bis = new BufferedInputStream(fileInputStream, 512000) ) {
+      return importList(aFormat, bis);
     }
   }
 
@@ -94,8 +96,9 @@ public Optional<Filmlist> importList(final FilmlistFormats aFormat, final URL aU
       if (responseBody == null) {
         return Optional.empty();
       }
-      try (final InputStream fileInputStream = responseBody.byteStream()) {
-        return importList(aFormat, fileInputStream);
+      try (final InputStream fileInputStream = responseBody.byteStream();
+          final BufferedInputStream bis = new BufferedInputStream(fileInputStream, 512000)) {
+        return importList(aFormat, bis);
       }
     }
   }

src/main/java/de/mediathekview/mlib/filmlisten/reader/AbstractFilmlistReader.java

@@ -9,12 +9,12 @@
 
 public abstract class AbstractFilmlistReader extends MessageCreator
 {
-    public AbstractFilmlistReader()
+    protected AbstractFilmlistReader()
     {
         super();
     }
 
-    public AbstractFilmlistReader(final MessageListener... aListeners)
+    protected AbstractFilmlistReader(final MessageListener... aListeners)
     {
         super(aListeners);
     }

src/main/java/de/mediathekview/mlib/filmlisten/reader/FilmlistOldFormatReader.java

@@ -99,6 +99,9 @@ private List<Future<Film>> asyncConvertEntriesToFilms(
       } else if (splittedEntry.size() == 21 && FILM_ENTRY_ID.equals(splittedEntry.get(0))) {
         filmEntryBefore =
             convertEntryToFilm(filmEntryBefore, executorService, futureFilms, splittedEntry);
+      } else {
+        // TODO do something useful here  
+        LOG.debug(String.format("unkown entry %s",splittedEntry));
       }
     }
     return futureFilms;
@@ -145,9 +148,9 @@ private void setMetaInfo(final Filmlist aFilmlist, final List<String> aSplittedE
 
   private void setListId(final Filmlist aFilmlist, final List<String> aSplittedEntry) {
     try {
-      aFilmlist.setListId(UUID.fromString(aSplittedEntry.get(4)));
+      aFilmlist.setListId(UUID.fromString(aSplittedEntry.get(5)));
     } catch (final IllegalArgumentException illegalArgumentException) {
-      LOG.debug("Can't parse the film list id. Setting a random uuid.", illegalArgumentException);
+      LOG.debug(String.format("Can't parse the film list id. Setting a random uuid. %s", aSplittedEntry), illegalArgumentException);
       aFilmlist.setListId(UUID.randomUUID());
     }
   }