Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

bugfix 992-991-990-989 and remove java preview feature #994

Merged
merged 2 commits into from
May 25, 2024
Merged

bugfix 992-991-990-989 and remove java preview feature #994

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
e846616
fix 992-01-990-989 and remove java preview feature
codingPF May 24, 2024
e2e4203
update test cases
codingPF May 24, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
12 changes: 12 additions & 0 deletions src/main/java/de/mediathekview/mserver/crawler/ard/ArdConstants.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -3,6 +3,7 @@
import java.util.HashMap;
import java.util.Map;

import de.mediathekview.mlib.daten.Resolution;
import de.mediathekview.mlib.daten.Sender;

public class ArdConstants {
Expand Down Expand Up @@ -67,6 +68,17 @@ public class ArdConstants {
public static final String WEBSITE_URL = "https://www.ardmediathek.de/video/%s";

public static final String BASE_URL_SUBTITLES = "https://classic.ardmediathek.de";

// provide the same as master until crawler release
public static Resolution getResolutionFromWidth(final int width) {
if (width > 1280) {
return Resolution.HD;
} else if (width > 640) {
return Resolution.NORMAL;
} else {
return Resolution.SMALL;
}
}

private ArdConstants() {}
}
60 changes: 43 additions & 17 deletions src/main/java/de/mediathekview/mserver/crawler/ard/json/ArdFilmDeserializer.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -7,6 +7,7 @@
import de.mediathekview.mlib.daten.Sender;
import de.mediathekview.mserver.base.utils.GeoLocationGuesser;
import de.mediathekview.mserver.base.utils.JsonUtils;
import de.mediathekview.mserver.base.utils.UrlUtils;
import de.mediathekview.mserver.crawler.ard.ArdConstants;
import de.mediathekview.mserver.crawler.ard.ArdFilmDto;
import de.mediathekview.mserver.crawler.ard.ArdFilmInfoDto;
Expand Down Expand Up @@ -57,13 +58,16 @@ public class ArdFilmDeserializer implements JsonDeserializer<List<ArdFilmDto>> {
private static final String ATTRIBUTE_RESOLUTION_H = "maxHResolutionPx";
private static final String ATTRIBUTE_MIME = "mimeType";
private static final String ATTRIBUTE_KIND = "kind";
private static final String ATTRIBUTE_ADUIO_LANG = "languageCode";

private static final String MARKER_VIDEO_MP4 = "video/mp4";
private static final String MARKER_VIDEO_STANDARD = "standard";
private static final String MARKER_VIDEO_CATEGORY_MAIN = "main";
private static final String MARKER_VIDEO_CATEGORY_MPEG = "application/vnd.apple.mpegurl";
private static final String MARKER_VIDEO_AD = "audio-description";
private static final String MARKER_VIDEO_DGS = "sign-language";
private static final String MARKER_VIDEO_DGS = "sign-language";
private static final String MARKER_VIDEO_OV = "OV";
private static final String MARKER_VIDEO_DE = "deu";

private final ArdVideoInfoJsonDeserializer videoDeserializer;
private final AbstractCrawler crawler;
Expand Down Expand Up @@ -292,7 +296,6 @@ private Film createFilm(
duration == null ? Duration.ofSeconds(0) : duration);

Optional.ofNullable(description).ifPresent(film::setBeschreibung);


film.setGeoLocations(GeoLocationGuesser.getGeoLocations(Sender.ARD, videoInfo.getDefaultVideoUrl()));

Expand Down Expand Up @@ -354,12 +357,19 @@ private void addDGSUrls(final Film film, final Map<Resolution, String> videoUrls
private Optional<ArdVideoInfoDto> parseVideos(final JsonObject playerPageObject, final String title) {
ArdVideoInfoDto allVideoUrls = new ArdVideoInfoDto();
//
final Optional<Map<Resolution, String>> videoInfoStandard = parseVideoUrls(playerPageObject, MARKER_VIDEO_CATEGORY_MAIN, MARKER_VIDEO_STANDARD, MARKER_VIDEO_MP4);
final Optional<Map<Resolution, String>> videoInfoAdaptive = parseVideoUrls(playerPageObject, MARKER_VIDEO_CATEGORY_MAIN, MARKER_VIDEO_STANDARD, MARKER_VIDEO_CATEGORY_MPEG);
final Optional<Map<Resolution, String>> videoInfoAD = parseVideoUrls(playerPageObject, MARKER_VIDEO_CATEGORY_MAIN, MARKER_VIDEO_AD, MARKER_VIDEO_MP4);
final Optional<Map<Resolution, String>> videoInfoDGS = parseVideoUrls(playerPageObject, MARKER_VIDEO_DGS, MARKER_VIDEO_STANDARD, MARKER_VIDEO_MP4);
final Optional<Map<Resolution, String>> videoInfoStandard = parseVideoUrls(playerPageObject, MARKER_VIDEO_CATEGORY_MAIN, MARKER_VIDEO_STANDARD, MARKER_VIDEO_MP4, MARKER_VIDEO_DE);
final Optional<Map<Resolution, String>> videoInfoAdaptive = parseVideoUrls(playerPageObject, MARKER_VIDEO_CATEGORY_MAIN, MARKER_VIDEO_STANDARD, MARKER_VIDEO_CATEGORY_MPEG, MARKER_VIDEO_DE);
final Optional<Map<Resolution, String>> videoInfoAD = parseVideoUrls(playerPageObject, MARKER_VIDEO_CATEGORY_MAIN, MARKER_VIDEO_AD, MARKER_VIDEO_MP4, MARKER_VIDEO_DE);
final Optional<Map<Resolution, String>> videoInfoDGS = parseVideoUrls(playerPageObject, MARKER_VIDEO_DGS, MARKER_VIDEO_STANDARD, MARKER_VIDEO_MP4, MARKER_VIDEO_DE);
final Optional<Map<Resolution, String>> videoInfoOV = parseVideoUrls(playerPageObject, MARKER_VIDEO_CATEGORY_MAIN, MARKER_VIDEO_STANDARD, MARKER_VIDEO_MP4, MARKER_VIDEO_OV);
final Optional<Set<String>> subtitles = prepareSubtitleUrl(playerPageObject);
//
// OV is a single film but also included in the standard film
if ((title.toUpperCase().contains("(OV)") || title.toUpperCase().contains("(ORIGINALVERSION)")) && videoInfoOV.isPresent()) {
allVideoUrls.putAll(videoInfoOV.get());
return Optional.of(allVideoUrls);
}
//
if (subtitles.isPresent()) {
allVideoUrls.setSubtitleUrl(subtitles);
}
Expand All @@ -385,16 +395,38 @@ private Optional<ArdVideoInfoDto> parseVideos(final JsonObject playerPageObject,
}
return Optional.of(allVideoUrls);
}

private Optional<Map<Resolution, String>> parseVideoUrls(final JsonObject playerPageObject, String streamType, String aduioType, String mimeType, String language) {
Optional<Map<Integer, String>> urls = parseVideoUrlMap(playerPageObject, streamType, aduioType, mimeType, language);
if (urls.isEmpty()) {
return Optional.empty();
}
Map<Resolution, String> videoInfo = new EnumMap<>(Resolution.class);
for (Map.Entry<Integer, String> entry : urls.get().entrySet()) {
Resolution resolution = ArdConstants.getResolutionFromWidth(entry.getKey());
if(!videoInfo.containsKey(resolution)) {
videoInfo.put(resolution, entry.getValue());
}
}
// issue if we do not have normal res
// TODO: FIXME
if (!videoInfo.containsKey(Resolution.NORMAL)) {
Resolution anyResolution = videoInfo.keySet().stream().findFirst().get();
videoInfo.put(Resolution.NORMAL, videoInfo.get(anyResolution));
videoInfo.remove(anyResolution);
}
return Optional.of(videoInfo);
}

private Optional<Map<Resolution, String>> parseVideoUrls(final JsonObject playerPageObject, String streamType, String aduioType, String mimeType) {
private Optional<Map<Integer, String>> parseVideoUrlMap(final JsonObject playerPageObject, String streamType, String aduioType, String mimeType, String language) {
final Optional<JsonObject> mediaCollectionObject = getMediaCollectionObject(playerPageObject);
if (mediaCollectionObject.isEmpty())
return Optional.empty();
final Optional<JsonElement> streams = JsonUtils.getElement(mediaCollectionObject.get(), ELEMENT_STREAMS);
if (streams.isEmpty() || !streams.get().isJsonArray() || (streams.get().getAsJsonArray().size() == 0))
return Optional.empty();
//
Map<Resolution, String> videoInfo = new EnumMap<>(Resolution.class);
Map<Integer, String> videoInfo = new TreeMap<>(Comparator.reverseOrder());
for (JsonElement streamsCategory : streams.get().getAsJsonArray()) {
final Optional<String> streamKind = JsonUtils.getElementValueAsString(streamsCategory, ATTRIBUTE_KIND);
final Optional<JsonElement> media = JsonUtils.getElement(streamsCategory, ELEMENT_MEDIA);
Expand All @@ -409,9 +441,9 @@ private Optional<Map<Resolution, String>> parseVideoUrls(final JsonObject player
Optional<String> kind = JsonUtils.getElementValueAsString(audios.get().getAsJsonArray().get(0), ATTRIBUTE_KIND);
Optional<String> resh = JsonUtils.getElementValueAsString(video, ATTRIBUTE_RESOLUTION_H);
Optional<String> url = JsonUtils.getElementValueAsString(video, ATTRIBUTE_URL);
if (url.isPresent() && resh.isPresent() && kind.isPresent() && kind.get().equalsIgnoreCase(aduioType)) {
Resolution resolution = Resolution.getResolutionFromWidth(Integer.parseInt(resh.get()));
videoInfo.put(resolution, url.get());
Optional<String> languageCode = JsonUtils.getElementValueAsString(audios.get().getAsJsonArray().get(0), ATTRIBUTE_ADUIO_LANG);
if (url.isPresent() && resh.isPresent() && kind.isPresent() && kind.get().equalsIgnoreCase(aduioType) && languageCode.orElse("").equalsIgnoreCase(language)) {
videoInfo.put(Integer.parseInt(resh.get()), UrlUtils.removeParameters(url.get()));
}
}
}
Expand All @@ -421,12 +453,6 @@ private Optional<Map<Resolution, String>> parseVideoUrls(final JsonObject player
if (videoInfo.size() == 0) {
return Optional.empty();
}
// TODO: FIX ME
if (!videoInfo.containsKey(Resolution.NORMAL)) {
Resolution r = videoInfo.keySet().stream().findFirst().get();
videoInfo.put(Resolution.NORMAL, videoInfo.get(r));
videoInfo.remove(r);
}
return Optional.of(videoInfo);
}
}
3 changes: 2 additions & 1 deletion .../java/de/mediathekview/mserver/crawler/ard/json/ArdMediaArrayToDownloadUrlsConverter.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -145,7 +145,8 @@ private static Resolution getQualityForNumber(final int qualityIndicator) {
case 1 -> Resolution.SMALL;
case 3, 4 -> Resolution.HD;
case 5 -> Resolution.UHD;
case 2, default -> Resolution.NORMAL;
case 2 -> Resolution.NORMAL;
default -> Resolution.NORMAL;
};
}

Expand Down
30 changes: 14 additions & 16 deletions ...main/java/de/mediathekview/mserver/crawler/funk/tasks/NexxCloudSessionInitiationTask.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -37,24 +37,22 @@ public NexxCloudSessionInitiationTask(final AbstractCrawler aCrawler) {
public Long call() {
final Gson gson = createGson();

try(final Client client = createClient()) {
final WebTarget target =
client.target(FunkUrls.NEXX_CLOUD_SESSION_INIT.getAsString(crawler.getRuntimeConfig()));
final Client client = createClient();
final WebTarget target =
client.target(FunkUrls.NEXX_CLOUD_SESSION_INIT.getAsString(crawler.getRuntimeConfig()));

final MultivaluedHashMap<String, String> formData = createForm();
final MultivaluedHashMap<String, String> formData = createForm();

try(final Response response = target.request().post(Entity.form(formData)))
{
if (response.getStatus() == 201) {
final String jsonOutput = response.readEntity(String.class);
return gson.fromJson(jsonOutput, Long.class);
} else {
crawler.printErrorMessage();
LOG.fatal(
"A HTTP error {} occurred when initialising the Nexx cloud session.",
response.getStatus());
}
}
final Response response = target.request().post(Entity.form(formData));

if (response.getStatus() == 201) {
final String jsonOutput = response.readEntity(String.class);
return gson.fromJson(jsonOutput, Long.class);
} else {
crawler.printErrorMessage();
LOG.fatal(
"A HTTP error {} occurred when initialising the Nexx cloud session.",
response.getStatus());
}
return null;
}
Expand Down
3 changes: 3 additions & 0 deletions src/main/java/de/mediathekview/mserver/crawler/zdf/json/ZdfFilmDetailDeserializer.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -303,6 +303,9 @@ private Optional<String> formatEpisodeTitle(Optional<Integer> season, Optional<I
if (season.isPresent()) {
result += String.format("S%02d", season.get());
}
if (season.isPresent() && episode.isPresent()) {
result += "/";
}
if (episode.isPresent()) {
result += String.format("E%02d", episode.get());
}
Expand Down
Loading
Loading