Merge pull request #881 from mediathekview/feature/zdf_hd

zdf: support uhd+fhd
mediathekview · Feb 15, 2023 · 8b107b8 · 8b107b8
2 parents e3dbdd4 + 0966082
commit 8b107b8
Show file tree

Hide file tree

Showing 14 changed files with 1,191 additions and 94 deletions.
diff --git a/src/main/java/de/mediathekview/mserver/base/utils/JsonUtils.java b/src/main/java/de/mediathekview/mserver/base/utils/JsonUtils.java
@@ -65,6 +65,17 @@ public static Optional<String> getAttributeAsString(
     return Optional.empty();
   }
 
+  public static Optional<Integer> getAttributeAsInt(final JsonObject jsonObject, final String attributeName) {
+    if (jsonObject.has(attributeName)) {
+      final JsonElement aElement = jsonObject.get(attributeName);
+      if (!aElement.isJsonNull()) {
+        return Optional.of(aElement.getAsInt());
+      }
+    }
+
+    return Optional.empty();
+  }
+
   public static Optional<String> getElementValueAsString(final JsonElement aJsonElement, final String... aElementIds) {
     Optional<String> rs = Optional.empty();
     JsonObject aJsonObject = aJsonElement.getAsJsonObject();

diff --git a/src/main/java/de/mediathekview/mserver/crawler/zdf/ZdfVideoUrlOptimizer.java b/src/main/java/de/mediathekview/mserver/crawler/zdf/ZdfVideoUrlOptimizer.java
@@ -32,17 +32,22 @@ public class ZdfVideoUrlOptimizer {
   private static final String NORMAL_2360_35_17 = "2360k_p35v17.mp4";
 
   private static final String HD_3256 = "3256k_p15v12.mp4";
-  private static final String HD_3296 = "3296k_p15v13.mp4";
+  private static final String HD_3296_15_13 = "3296k_p15v13.mp4";
+  private static final String HD_3296_15_14 = "3296k_p15v14.mp4";
+  private static final String HD_3328_15_15 = "3328k_p15v15.mp4";
   private static final String HD_3328_12 = "3328k_p36v12.mp4";
   private static final String HD_3328_13 = "3328k_p36v13.mp4";
   private static final String HD_3328_14 = "3328k_p36v14.mp4";
   private static final String HD_3328_35_14 = "3328k_p35v14.mp4";
+  private static final String HD_3328_36_13 = "3328k_p36v13.mp4";
+  private static final String HD_3328_36_14 = "3328k_p36v14.mp4";
   private static final String HD_3360_36_15 = "3360k_p36v15.mp4";
   private static final String HD_3360_36_17 = "3360k_p36v17.mp4";
   private static final String HD_6628_61_17 = "6628k_p61v17.mp4";
   private static final String HD_6660_37_17 = "6660k_p37v17.mp4";
 
   private static final Map<String, String[]> NORMAL_OPTIMIZE = new HashMap<>();
+  private static final Map<String, String[]> NORMAL_TO_HD = new HashMap<>();
   private static final Map<String, String[]> HD_OPTIMIZE = new HashMap<>();
 
   static {
@@ -57,20 +62,26 @@ public class ZdfVideoUrlOptimizer {
     NORMAL_OPTIMIZE.put(NORMAL_1496_13_14, new String[] {NORMAL_2328_35_14, NORMAL_2296_14_14});
     NORMAL_OPTIMIZE.put(NORMAL_1628_13_15, new String[] {NORMAL_2360_35_15});
 
-
-    HD_OPTIMIZE.put(NORMAL_2360_35_17, new String[] {HD_6660_37_17, HD_6628_61_17, HD_3360_36_17});
-    HD_OPTIMIZE.put(NORMAL_1628_13_17, new String[] {HD_6660_37_17, HD_6628_61_17, HD_3360_36_17});
-    HD_OPTIMIZE.put(NORMAL_1456_13_12, new String[] {HD_3328_12, HD_3256});
-    HD_OPTIMIZE.put(NORMAL_2256_14_12, new String[] {HD_3328_12, HD_3256});
-    HD_OPTIMIZE.put(NORMAL_2328_35_12, new String[] {HD_3328_12, HD_3256});
-    HD_OPTIMIZE.put(NORMAL_1496_13_13, new String[] {HD_3328_13, HD_3296});
-    HD_OPTIMIZE.put(NORMAL_2296_14_13, new String[] {HD_3328_13, HD_3296});
-    HD_OPTIMIZE.put(NORMAL_2328_35_13, new String[] {HD_3328_13, HD_3296});
-    HD_OPTIMIZE.put(NORMAL_1496_13_14, new String[] {HD_3328_14, HD_3328_35_14});
-    HD_OPTIMIZE.put(NORMAL_2296_14_14, new String[] {HD_3328_14, HD_3328_35_14});
-    HD_OPTIMIZE.put(NORMAL_2328_35_14, new String[] {HD_3328_14, HD_3328_35_14});
-    HD_OPTIMIZE.put(NORMAL_1628_13_15, new String[] {HD_3360_36_15});
-    HD_OPTIMIZE.put(NORMAL_2360_35_15, new String[] {HD_3360_36_15});
+    NORMAL_TO_HD.put(NORMAL_2360_35_17, new String[] {HD_6660_37_17, HD_6628_61_17, HD_3360_36_17});
+    NORMAL_TO_HD.put(NORMAL_1628_13_17, new String[] {HD_6660_37_17, HD_6628_61_17, HD_3360_36_17});
+    NORMAL_TO_HD.put(NORMAL_1456_13_12, new String[] {HD_3328_12, HD_3256});
+    NORMAL_TO_HD.put(NORMAL_2256_14_12, new String[] {HD_3328_12, HD_3256});
+    NORMAL_TO_HD.put(NORMAL_2328_35_12, new String[] {HD_3328_12, HD_3256});
+    NORMAL_TO_HD.put(NORMAL_1496_13_13, new String[] {HD_3328_13, HD_3296_15_14, HD_3296_15_13});
+    NORMAL_TO_HD.put(NORMAL_2296_14_13, new String[] {HD_3328_13, HD_3296_15_14, HD_3296_15_13});
+    NORMAL_TO_HD.put(NORMAL_2328_35_13, new String[] {HD_3328_13, HD_3296_15_14, HD_3296_15_13});
+    NORMAL_TO_HD.put(NORMAL_1496_13_14, new String[] {HD_3328_14, HD_3328_35_14});
+    NORMAL_TO_HD.put(NORMAL_2296_14_14, new String[] {HD_3328_14, HD_3328_35_14});
+    NORMAL_TO_HD.put(NORMAL_2328_35_14, new String[] {HD_3328_14, HD_3328_35_14});
+    NORMAL_TO_HD.put(NORMAL_1628_13_15, new String[] {HD_3360_36_15});
+    NORMAL_TO_HD.put(NORMAL_2360_35_15, new String[] {HD_3360_36_15});
+
+    HD_OPTIMIZE.put(HD_3360_36_17, new String[] {HD_6660_37_17, HD_6628_61_17});
+    HD_OPTIMIZE.put(HD_6628_61_17, new String[] {HD_6660_37_17});
+    HD_OPTIMIZE.put(HD_3328_15_15, new String[] {HD_3360_36_15});
+    HD_OPTIMIZE.put(HD_3256, new String[] {HD_3328_12});
+    HD_OPTIMIZE.put(HD_3296_15_14, new String[] {HD_3328_36_14});
+    HD_OPTIMIZE.put(HD_3296_15_13, new String[] {HD_3328_36_13});
   }
 
   protected AbstractCrawler crawler;
@@ -89,6 +100,15 @@ public String getOptimizedUrlNormal(final String aUrl) {
     return optimize(aUrl, NORMAL_OPTIMIZE);
   }
 
+  /**
+   * optimizes the hd url
+   * @param aUrl the hd url
+   * @return the optimized hd url
+   */
+  public String getOptimizedUrlHd(final String aUrl) {
+    return optimize(aUrl, HD_OPTIMIZE);
+  }
+
   /**
    * tries to find the hd url depending on the normal url.
    *
@@ -100,7 +120,7 @@ public Optional<String> determineUrlHd(final String aNormalUrl) {
       return Optional.empty();
     }
 
-    final String url = optimize(aNormalUrl, HD_OPTIMIZE);
+    final String url = optimize(aNormalUrl, NORMAL_TO_HD);
     if (url.equalsIgnoreCase(aNormalUrl)) {
       return Optional.empty();
     }

diff --git a/src/main/java/de/mediathekview/mserver/crawler/zdf/json/ZdfDownloadDtoDeserializer.java b/src/main/java/de/mediathekview/mserver/crawler/zdf/json/ZdfDownloadDtoDeserializer.java
@@ -1,25 +1,26 @@
 package de.mediathekview.mserver.crawler.zdf.json;
 
-import com.google.gson.JsonArray;
-import com.google.gson.JsonDeserializationContext;
-import com.google.gson.JsonDeserializer;
-import com.google.gson.JsonElement;
-import com.google.gson.JsonObject;
+import com.google.gson.*;
 import de.mediathekview.mlib.daten.GeoLocations;
 import de.mediathekview.mlib.daten.Resolution;
-import java.lang.reflect.Type;
-import java.time.Duration;
-import java.util.Iterator;
-import java.util.Optional;
+import de.mediathekview.mserver.base.utils.JsonUtils;
 import org.apache.logging.log4j.LogManager;
 import org.apache.logging.log4j.Logger;
 
+import java.lang.reflect.Type;
+import java.time.Duration;
+import java.util.*;
+
 /** A JSON deserializer to gather the needed information for a {@link DownloadDto}. */
 public class ZdfDownloadDtoDeserializer implements JsonDeserializer<Optional<DownloadDto>> {
 
+  private static final String ZDF_QUALITY_UHD = "uhd";
+  private static final String ZDF_QUALITY_FHD = "fhd";
+  private static final String ZDF_QUALITY_HD = "hd";
   private static final String ZDF_QUALITY_VERYHIGH = "veryhigh";
   private static final String ZDF_QUALITY_HIGH = "high";
   private static final String ZDF_QUALITY_MED = "med";
+  private static final String ZDF_QUALITY_MEDIUM = "medium";
   private static final String ZDF_QUALITY_LOW = "low";
   private static final Logger LOG = LogManager.getLogger(ZdfDownloadDtoDeserializer.class);
   private static final String JSON_ELEMENT_ATTRIBUTES = "attributes";
@@ -29,7 +30,7 @@ public class ZdfDownloadDtoDeserializer implements JsonDeserializer<Optional<Dow
   private static final String JSON_ELEMENT_DURATION = "duration";
   private static final String JSON_ELEMENT_FORMITAET = "formitaeten";
   private static final String JSON_ELEMENT_GEOLOCATION = "geoLocation";
-  private static final String JSON_ELEMENT_HD = "hd";
+  private static final String JSON_ELEMENT_HIGHEST_VERTIVAL_RESOLUTION = "highestVerticalResolution";
   private static final String JSON_ELEMENT_LANGUAGE = "language";
   private static final String JSON_ELEMENT_MIMETYPE = "mimeType";
   private static final String JSON_ELEMENT_PRIORITYLIST = "priorityList";
@@ -45,6 +46,24 @@ public class ZdfDownloadDtoDeserializer implements JsonDeserializer<Optional<Dow
   private static final String RELEVANT_SUBTITLE_TYPE = ".xml";
   private static final String JSON_ELEMENT_QUALITIES = "qualities";
 
+  private static AbstractMap.SimpleEntry<String, String> extractTrack(
+          JsonElement aTrackElement) {
+    JsonObject trackObject = aTrackElement.getAsJsonObject();
+    String classValue = trackObject.get(JSON_ELEMENT_CLASS).getAsString();
+    String language = trackObject.get(JSON_ELEMENT_LANGUAGE).getAsString();
+    String uri = trackObject.get(JSON_ELEMENT_URI).getAsString();
+
+    // films with audiodescription are handled as a language
+    if (CLASS_AD.equalsIgnoreCase(classValue)) {
+      language += "-ad";
+    }
+    if (uri != null) {
+      return new AbstractMap.SimpleEntry<>(language, uri);
+    } else {
+      throw new RuntimeException("uri is null");
+    }
+  }
+
   @Override
   public Optional<DownloadDto> deserialize(
       final JsonElement aJsonElement,
@@ -73,7 +92,8 @@ private void parseDuration(final DownloadDto dto, final JsonObject rootNode) {
     if (attributes != null) {
       final JsonElement durationElement = attributes.getAsJsonObject().get(JSON_ELEMENT_DURATION);
       if (durationElement != null) {
-        final JsonElement durationValue = durationElement.getAsJsonObject().get(JSON_PROPERTY_VALUE);
+        final JsonElement durationValue =
+            durationElement.getAsJsonObject().get(JSON_PROPERTY_VALUE);
         if (durationValue != null) {
           dto.setDuration(Duration.ofMillis(durationValue.getAsLong()));
         }
@@ -86,13 +106,15 @@ private void parseFormitaet(final DownloadDto dto, final JsonElement formitaet)
     final JsonElement mimeType = formitaet.getAsJsonObject().get(JSON_ELEMENT_MIMETYPE);
     if (mimeType != null && mimeType.getAsString().equalsIgnoreCase(RELEVANT_MIME_TYPE)) {
 
+      List<DownloadInfo> downloads = new ArrayList<>();
+
       // array Resolution
       final JsonArray qualityList =
           formitaet.getAsJsonObject().getAsJsonArray(JSON_ELEMENT_QUALITIES);
       for (final JsonElement quality : qualityList) {
 
-        final Resolution qualityValue = parseVideoQuality(quality.getAsJsonObject());
-
+        final Resolution resolution = parseVideoQuality(quality.getAsJsonObject());
+        final Optional<Integer> verticalResolution = JsonUtils.getAttributeAsInt(quality.getAsJsonObject(), JSON_ELEMENT_HIGHEST_VERTIVAL_RESOLUTION);
         // subelement audio
         final JsonElement audio = quality.getAsJsonObject().get(JSON_ELEMENT_AUDIO);
         if (audio != null) {
@@ -101,28 +123,14 @@ private void parseFormitaet(final DownloadDto dto, final JsonElement formitaet)
           final JsonArray tracks = audio.getAsJsonObject().getAsJsonArray(JSON_ELEMENT_TRACKS);
 
           for (JsonElement trackElement : tracks) {
-            extractTrack(dto, qualityValue, trackElement);
+            final AbstractMap.SimpleEntry<String, String> languageUri = extractTrack(trackElement);
+            downloads.add(new DownloadInfo(languageUri.getKey(), languageUri.getValue(), verticalResolution.orElse(0), resolution));
           }
         }
       }
-    }
-  }
-
-  private static void extractTrack(
-      DownloadDto aDto, Resolution aQualityValue, JsonElement aTrackElement) {
-    JsonObject trackObject = aTrackElement.getAsJsonObject();
-    String classValue = trackObject.get(JSON_ELEMENT_CLASS).getAsString();
-    String language = trackObject.get(JSON_ELEMENT_LANGUAGE).getAsString();
-    String uri = trackObject.get(JSON_ELEMENT_URI).getAsString();
 
-    // films with audiodescription are handled as a language
-    if (CLASS_AD.equalsIgnoreCase(classValue)) {
-      language += "-ad";
-    }
-    if (aQualityValue != null && uri != null) {
-      aDto.addUrl(language, aQualityValue, uri);
-    } else {
-      throw new RuntimeException("either quality or uri is null");
+      downloads.sort(Comparator.comparingInt(DownloadInfo::verticalResolution));
+      downloads.forEach(info -> dto.addUrl(info.language(), info.resolution(), info.uri()));
     }
   }
 
@@ -137,7 +145,7 @@ private void parseGeoLocation(final DownloadDto dto, final JsonObject rootNode)
           if (foundGeoLocation.isPresent()) {
             dto.setGeoLocation(foundGeoLocation.get());
           } else {
-            LOG.debug(String.format("Can't find a GeoLocation for \"%s", geoValue.getAsString()));
+            LOG.debug("Can't find a GeoLocation for {}", geoValue.getAsString());
           }
         }
       }
@@ -180,27 +188,27 @@ private void parseSubtitle(final DownloadDto dto, final JsonObject rootNode) {
 
   private Resolution parseVideoQuality(final JsonObject quality) {
     Resolution qualityValue;
-    final JsonElement hd = quality.get(JSON_ELEMENT_HD);
-    if (hd != null && hd.getAsBoolean()) {
-      qualityValue = Resolution.HD;
-    } else {
-      final String zdfQuality = quality.get(JSON_ELEMENT_QUALITY).getAsString();
-      switch (zdfQuality) {
-        case ZDF_QUALITY_LOW:
-          qualityValue = Resolution.SMALL;
-          break;
-        case ZDF_QUALITY_MED:
-          qualityValue = Resolution.SMALL;
-          break;
-        case ZDF_QUALITY_HIGH:
-          qualityValue = Resolution.SMALL;
-          break;
-        case ZDF_QUALITY_VERYHIGH:
-          qualityValue = Resolution.NORMAL;
-          break;
-        default:
-          qualityValue = Resolution.VERY_SMALL;
-      }
+    final String zdfQuality = quality.get(JSON_ELEMENT_QUALITY).getAsString();
+    switch (zdfQuality) {
+      case ZDF_QUALITY_LOW:
+      case ZDF_QUALITY_MED:
+      case ZDF_QUALITY_MEDIUM:
+      case ZDF_QUALITY_HIGH:
+        qualityValue = Resolution.SMALL;
+        break;
+      case ZDF_QUALITY_VERYHIGH:
+        qualityValue = Resolution.NORMAL;
+        break;
+      case ZDF_QUALITY_HD:
+      case ZDF_QUALITY_FHD:
+        qualityValue = Resolution.HD;
+        break;
+      case ZDF_QUALITY_UHD:
+        qualityValue = Resolution.UHD;
+        break;
+      default:
+        LOG.error("unknown quality: {}", zdfQuality);
+        qualityValue = Resolution.VERY_SMALL;
     }
     return qualityValue;
   }
@@ -213,4 +221,7 @@ private void parseVideoUrls(final DownloadDto dto, final JsonObject rootNode) {
       parsePriority(dto, priority);
     }
   }
+
+  private record DownloadInfo(String language, String uri, int verticalResolution, Resolution resolution) {
+  }
 }
diff --git a/src/main/java/de/mediathekview/mserver/crawler/zdf/json/ZdfFilmDetailDeserializer.java b/src/main/java/de/mediathekview/mserver/crawler/zdf/json/ZdfFilmDetailDeserializer.java
@@ -46,7 +46,7 @@ public class ZdfFilmDetailDeserializer implements JsonDeserializer<Optional<ZdfF
   private static final String JSON_ATTRIBUTE_TEMPLATE = "http://zdf.de/rels/streams/ptmd-template";
 
   private static final String PLACEHOLDER_PLAYER_ID = "{playerId}";
-  private static final String PLAYER_ID = "ngplayer_2_3";
+  private static final String PLAYER_ID = "android_native_5";
 
   private static final String DOWNLOAD_URL_DEFAULT = "default";
   private static final String DOWNLOAD_URL_DGS = "dgs";

diff --git a/src/main/java/de/mediathekview/mserver/crawler/zdf/tasks/ZdfFilmDetailTask.java b/src/main/java/de/mediathekview/mserver/crawler/zdf/tasks/ZdfFilmDetailTask.java
@@ -205,6 +205,8 @@ private Map<Resolution, FilmUrl> getOptimizedUrls(Map<Resolution, String> urls)
 
       if (qualitiesEntry.getKey() == Resolution.NORMAL) {
         url = optimizer.getOptimizedUrlNormal(url);
+      } else if (qualitiesEntry.getKey() == Resolution.HD) {
+        url = optimizer.getOptimizedUrlHd(url);
       }
 
       result.put(qualitiesEntry.getKey(), new FilmUrl(url, crawler.determineFileSizeInKB(url)));

diff --git a/src/test/java/de/mediathekview/mserver/crawler/phoenix/tasks/PhoenixFilmDetailTaskTest.java b/src/test/java/de/mediathekview/mserver/crawler/phoenix/tasks/PhoenixFilmDetailTaskTest.java
@@ -92,7 +92,7 @@ public static Collection<Object[]> data() {
             "/phoenix/phoenix_film_detail2_reponse.json",
             "/php/mediaplayer/data/beitrags_details.php?id=2121010",
             "/phoenix/phoenix_film_detail2_beitrag_details.json",
-            "/tmd/2/ngplayer_2_3/vod/ptmd/phoenix/210416_phx_doku_awri_2",
+            "/tmd/2/android_native_5/vod/ptmd/phoenix/210416_phx_doku_awri_2",
             "/phoenix/phoenix_film_detail2_video.json",
             "Alles was Recht ist",
             "(2/5): Persönlichkeit oder Öffentlichkeit?",
@@ -111,7 +111,7 @@ public static Collection<Object[]> data() {
             "/phoenix/phoenix_film_detail3_response.json",
             "/php/mediaplayer/data/beitrags_details.php?id=2361354",
             "/phoenix/phoenix_film_detail3_beitrag_details.json",
-            "/tmd/2/ngplayer_2_3/vod/ptmd/phoenix/211114_1200_phx_presseclub",
+            "/tmd/2/android_native_5/vod/ptmd/phoenix/211114_1200_phx_presseclub",
             "/phoenix/phoenix_film_detail3_video.json",
             "Presseclub",
             "Handeln statt Reden: Impfpflicht für alle?",

diff --git a/src/test/java/de/mediathekview/mserver/crawler/zdf/ZdfVideoUrlOptimizerTest.java b/src/test/java/de/mediathekview/mserver/crawler/zdf/ZdfVideoUrlOptimizerTest.java
@@ -57,6 +57,39 @@ public void getOptimizedUrlNormalTestBetterUrlNotExists() {
     assertGetOptimizedUrlNormal(url, url);
   }
 
+  @Test
+  public void getOptimizedUrlHdTestAlreadyBestQuality() {
+    final String url = getWireMockBaseUrlSafe() + "/video_6660k_p37v17.mp4";
+
+    assertGetOptimizedUrlHd(url, url);
+  }
+
+  @Test
+  public void getOptimizedUrlHdTestBetterUrlExists() {
+    final String url = getWireMockBaseUrlSafe() + "/video_3360k_p36v17.mp4";
+    final String expectedUrl = getWireMockBaseUrlSafe() + "/video_6660k_p37v17.mp4";
+
+    setupHeadResponse("/video_6660k_p37v17.mp4", 200);
+    assertGetOptimizedUrlHd(expectedUrl, url);
+  }
+
+  @Test
+  public void getOptimizedUrlHdTestBetterUrlExists2() {
+    final String url = getWireMockBaseUrlSafe() + "/video_3360k_p36v17.mp4";
+    final String expectedUrl = getWireMockBaseUrlSafe() + "/video_6628k_p61v17.mp4";
+
+    setupHeadResponse("/video_6628k_p61v17.mp4", 200);
+    assertGetOptimizedUrlHd(expectedUrl, url);
+  }
+
+  @Test
+  public void getOptimizedUrlHdTestBetterUrlNotExists() {
+    final String url = getWireMockBaseUrlSafe() + "/video_3360k_p36v17.mp4";
+
+    setupHeadResponse("/video_6660k_p37v17.mp4", 404);
+    assertGetOptimizedUrlHd(url, url);
+  }
+
   @Test
   public void determineUrlHdTestFirstUrlExists() {
     final String url = getWireMockBaseUrlSafe() + "/video_1456k_p13v12.mp4";
@@ -125,6 +158,11 @@ private void assertGetOptimizedUrlNormal(final String aExpectedUrl, final String
     assertThat(actual, equalTo(aExpectedUrl));
   }
 
+  private void assertGetOptimizedUrlHd(final String aExpectedUrl, final String aUrlToCheck) {
+    final String actual = target.getOptimizedUrlHd(aUrlToCheck);
+    assertThat(actual, equalTo(aExpectedUrl));
+  }
+
   private void assertDetermineUrlHd(final Optional<String> aExpectedUrl, final String aUrlToCheck) {
     final Optional<String> actual = target.determineUrlHd(aUrlToCheck);
     assertThat(actual, equalTo(aExpectedUrl));