From bc69685d3a11058e47d1cabe6e18a8c12aea8c18 Mon Sep 17 00:00:00 2001 From: David Schulte Date: Sun, 11 Aug 2024 08:41:32 +0200 Subject: [PATCH] feat: add support for native PGS subtitle rendering without transcoding (#2404) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-authored-by: Fernando Fernández --- frontend/package.json | 1 + frontend/src/store/playback-manager.ts | 17 +++++++-- frontend/src/store/player-element.ts | 37 +++++++++++++++++++ .../playback-profiles/subtitle-profile.ts | 4 ++ package-lock.json | 7 ++++ 5 files changed, 63 insertions(+), 3 deletions(-) diff --git a/frontend/package.json b/frontend/package.json index 9366b5156d4..1188c8cf604 100644 --- a/frontend/package.json +++ b/frontend/package.json @@ -36,6 +36,7 @@ "fast-equals": "5.0.1", "hls.js": "1.5.14", "jassub": "1.7.16", + "libpgs": "0.4.1", "lodash-es": "4.17.21", "marked": "14.0.0", "sortablejs": "1.15.2", diff --git a/frontend/src/store/playback-manager.ts b/frontend/src/store/playback-manager.ts index 0062caa3b2e..ba417a79e00 100644 --- a/frontend/src/store/playback-manager.ts +++ b/frontend/src/store/playback-manager.ts @@ -291,7 +291,7 @@ class PlaybackManagerStore extends CommonStore { srcLang: sub.Language ?? undefined, type: sub.DeliveryMethod ?? SubtitleDeliveryMethod.Drop, srcIndex: sub.srcIndex, - codec: sub.Codec === null ? undefined : sub.Codec + codec: sub.Codec === null ? undefined : sub.Codec?.toLowerCase() })); } } @@ -299,7 +299,7 @@ class PlaybackManagerStore extends CommonStore { /** * Filters the native subtitles * - * As our profile requires either SSA or VTT, if it's not SSA it'll be VTT. + * As our profile requires either SSA, PGS or VTT, if it's not SSA or PGS it'll be VTT. * This is done this way as server sends as "Codec" the initial value of the track, so it can be webvtt, subrip, srt... * This is easier to filter out the SSA subs */ @@ -307,7 +307,7 @@ class PlaybackManagerStore extends CommonStore { return ( this.currentItemParsedSubtitleTracks?.filter( (sub): sub is PlaybackExternalTrack => - !!sub.codec && sub.codec !== 'ass' && sub.codec !== 'ssa' && !!sub.src + !!sub.codec && sub.codec !== 'ass' && sub.codec !== 'ssa' && sub.codec !== 'pgssub' && !!sub.src ) ?? [] ); } @@ -323,6 +323,17 @@ class PlaybackManagerStore extends CommonStore { ); } + public get currentItemPgsParsedSubtitleTracks(): PlaybackExternalTrack[] { + return ( + this.currentItemParsedSubtitleTracks?.filter( + (sub): sub is PlaybackExternalTrack => + !!sub.codec + && (sub.codec === 'pgssub') + && !!sub.src + ) ?? [] + ); + } + public get currentVideoTrack(): MediaStream | undefined { if ( !isNil(this._state.currentMediaSource?.MediaStreams) diff --git a/frontend/src/store/player-element.ts b/frontend/src/store/player-element.ts index 723d192698a..7102750d693 100644 --- a/frontend/src/store/player-element.ts +++ b/frontend/src/store/player-element.ts @@ -7,6 +7,8 @@ import JASSUB from 'jassub'; import jassubWorker from 'jassub/dist/jassub-worker.js?url'; import jassubWasmUrl from 'jassub/dist/jassub-worker.wasm?url'; +import { PgsRenderer } from 'libpgs'; +import pgssubWorker from 'libpgs/dist/libpgs.worker.js?url'; import { computed, nextTick, shallowRef, watch } from 'vue'; import { playbackManager } from './playback-manager'; import { isArray, isNil, sealed } from '@/utils/validation'; @@ -35,6 +37,7 @@ class PlayerElementStore extends CommonStore { */ private readonly _fullscreenVideoRoute = '/playback/video'; private _jassub: JASSUB | undefined; + private _pgssub: PgsRenderer | undefined; protected _storeKey = 'playerElement'; public readonly isStretched = computed({ @@ -105,6 +108,30 @@ class PlayerElementStore extends CommonStore { ); }; + private readonly _setPgsTrack = (trackSrc: string): void => { + if ( + !this._pgssub + && mediaElementRef.value + && mediaElementRef.value instanceof HTMLVideoElement + ) { + this._pgssub = new PgsRenderer({ + video: mediaElementRef.value, + subUrl: trackSrc, + workerUrl: pgssubWorker + }); + } else if (this._pgssub) { + this._pgssub.loadFromUrl(trackSrc); + } + }; + + private readonly _freePgsTrack = (): void => { + if (this._pgssub) { + this._pgssub.dispose(); + } + + this._pgssub = undefined; + }; + /** * Applies the current subtitle from the playbackManager store * @@ -128,6 +155,9 @@ class PlayerElementStore extends CommonStore { const ass = playbackManager.currentItemAssParsedSubtitleTracks.find( sub => sub.srcIndex === playbackManager.currentSubtitleStreamIndex ); + const pgs = playbackManager.currentItemPgsParsedSubtitleTracks.find( + sub => sub.srcIndex === playbackManager.currentSubtitleStreamIndex + ); const attachedFonts = playbackManager.currentMediaSource?.MediaAttachments?.filter(a => this._isSupportedFont(a.MimeType) @@ -155,6 +185,7 @@ class PlayerElementStore extends CommonStore { } this._freeSsaTrack(); + this._freePgsTrack(); if (vttIdx !== -1 && mediaElementRef.value.textTracks[vttIdx]) { /** @@ -166,6 +197,11 @@ class PlayerElementStore extends CommonStore { * If SSA, using Subtitle Opctopus */ this._setSsaTrack(ass.src, attachedFonts); + } else if (pgs?.src) { + /** + * If PGS, using libpgs to render + */ + this._setPgsTrack(pgs.src); } }; @@ -196,6 +232,7 @@ class PlayerElementStore extends CommonStore { watch(videoContainerRef, () => { if (!videoContainerRef.value) { this._freeSsaTrack(); + this._freePgsTrack(); } }, { flush: 'sync' }); diff --git a/frontend/src/utils/playback-profiles/subtitle-profile.ts b/frontend/src/utils/playback-profiles/subtitle-profile.ts index 6875e92de93..7ce886efc2c 100644 --- a/frontend/src/utils/playback-profiles/subtitle-profile.ts +++ b/frontend/src/utils/playback-profiles/subtitle-profile.ts @@ -27,6 +27,10 @@ export function getSubtitleProfiles(): SubtitleProfile[] { { Format: 'ssa', Method: SubtitleDeliveryMethod.External + }, + { + Format: 'pgssub', + Method: SubtitleDeliveryMethod.External } ); diff --git a/package-lock.json b/package-lock.json index fb7eda254dd..e25bf1509bd 100644 --- a/package-lock.json +++ b/package-lock.json @@ -36,6 +36,7 @@ "fast-equals": "5.0.1", "hls.js": "1.5.14", "jassub": "1.7.16", + "libpgs": "0.4.1", "lodash-es": "4.17.21", "marked": "14.0.0", "sortablejs": "1.15.2", @@ -7628,6 +7629,12 @@ "node": ">= 0.8.0" } }, + "node_modules/libpgs": { + "version": "0.4.1", + "resolved": "https://registry.npmjs.org/libpgs/-/libpgs-0.4.1.tgz", + "integrity": "sha512-I4mIGz7Lf23xy/8mwSx0qlStz0oZFCz9dLC1xXNaqv5MbVdFhZWE+OMhVBLGjfVkjugyboM9XJ+4bCSibAIGuA==", + "license": "MIT" + }, "node_modules/lightningcss": { "version": "1.26.0", "resolved": "https://registry.npmjs.org/lightningcss/-/lightningcss-1.26.0.tgz",