From 41d55f22803ce8653336d0cbe8f6f8d7d0529944 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Mar=C3=ADa=20A=2E=20Matienzo?= <matienzo@berkeley.edu>
Date: Tue, 2 Jul 2024 17:56:20 -0700
Subject: [PATCH] handle WebVTT captions in HLS manifests

* add new helper methods to grab the URI for a WebVTT file included in a HLS manifest
* insert new <track/> tags specifically for HLS streams
* inject HLS.js configuration options to hide captions it renders

this will require setting the  property in all
wowza collections to
---
 Gemfile                                      |  1 +
 Gemfile.lock                                 |  3 +++
 app/views/player/_player_head_additional.erb |  3 +++
 app/views/player/_video.html.erb             |  4 +++-
 lib/av_player/track_extensions.rb            | 22 ++++++++++++++++++++
 spec/data/breslin1-playlist.m3u8             |  5 +++++
 spec/data/subtitlelist_leng_w289808588.m3u8  |  7 +++++++
 spec/lib/av_player/track_extensions_spec.rb  | 19 +++++++++++++++++
 8 files changed, 63 insertions(+), 1 deletion(-)
 create mode 100644 spec/data/breslin1-playlist.m3u8
 create mode 100644 spec/data/subtitlelist_leng_w289808588.m3u8

diff --git a/Gemfile b/Gemfile
index ec1b1af..e8a8a62 100644
--- a/Gemfile
+++ b/Gemfile
@@ -10,6 +10,7 @@ gem 'berkeley_library-docker', '~> 0.2.0'
 gem 'berkeley_library-logging', '~> 0.2'
 gem 'browser', '~> 4.2'
 gem 'jbuilder', '~> 2.7'
+gem 'm3u8', '~> 0.8.2'
 gem 'non-stupid-digest-assets', '~> 1.0' # Allow static pages (e.g. 404.html) to link to compiled assets
 gem 'omniauth-cas', '~> 2.0'
 gem 'puma', '~> 5.3', '>= 5.3.1'
diff --git a/Gemfile.lock b/Gemfile.lock
index 436ae8e..7fce552 100644
--- a/Gemfile.lock
+++ b/Gemfile.lock
@@ -149,6 +149,7 @@ GEM
     loofah (2.21.4)
       crass (~> 1.0.2)
       nokogiri (>= 1.12.0)
+    m3u8 (0.8.2)
     mail (2.8.1)
       mini_mime (>= 0.1.1)
       net-imap
@@ -352,6 +353,7 @@ PLATFORMS
   aarch64-linux
   arm64-darwin-21
   arm64-darwin-22
+  arm64-darwin-23
   x86_64-darwin-19
   x86_64-linux
 
@@ -368,6 +370,7 @@ DEPENDENCIES
   dotenv-rails
   jbuilder (~> 2.7)
   listen (>= 3.0)
+  m3u8 (~> 0.8.2)
   non-stupid-digest-assets (~> 1.0)
   omniauth-cas (~> 2.0)
   puma (~> 5.3, >= 5.3.1)
diff --git a/app/views/player/_player_head_additional.erb b/app/views/player/_player_head_additional.erb
index 18983c6..0da0a9b 100644
--- a/app/views/player/_player_head_additional.erb
+++ b/app/views/player/_player_head_additional.erb
@@ -34,6 +34,9 @@
 
       function initialize_players() {
           $("audio,video").mediaelementplayer({
+            // workaround for mediaelement/mediaelement#2963
+            // (when the HLS stream contains WebVTT captions)
+            hls: { enableWebVTT: false },
             iconSprite: "/assets/icons/mejs-controls.svg",
             success: function (mediaElement, originalNode, instance) {
                 if (typeof dashjs !== "undefined") {
diff --git a/app/views/player/_video.html.erb b/app/views/player/_video.html.erb
index faef104..4d40ed8 100644
--- a/app/views/player/_video.html.erb
+++ b/app/views/player/_video.html.erb
@@ -3,10 +3,12 @@
 <video id="video-<%= index %>" width="100%" height="270" preload="<%= preload %>" controls crossorigin="anonymous">
   <% if browser.platform.ios? || browser.device.ipad? %>
     <source src="<%= track.hls_uri %>" type="<%= BerkeleyLibrary::AV::Track::SOURCE_TYPE_HLS %>"/>
+    <% if (hls_vtt_uri = track.hls_vtt_uri) %>
+      <track kind="captions" srclang="en" src="<%= hls_vtt_uri %>"/>
+    <% end %>
   <% else %>
     <source src="<%= track.mpeg_dash_uri %>" type="<%= BerkeleyLibrary::AV::Track::SOURCE_TYPE_MPEG_DASH %>"/>
     <% if (dash_vtt_uri = track.dash_vtt_uri) %>
-      <%# TODO: should we also be adding a caption track for HLS streaming to iPhone/iPad? %>
       <track kind="captions" srclang="en" src="<%= dash_vtt_uri %>"/>
     <% end %>
   <% end %>
diff --git a/lib/av_player/track_extensions.rb b/lib/av_player/track_extensions.rb
index f93c111..b6aa860 100644
--- a/lib/av_player/track_extensions.rb
+++ b/lib/av_player/track_extensions.rb
@@ -22,6 +22,12 @@ def hls_uri
         @hls_uri ||= build_hls_uri
       end
 
+      def hls_vtt_uri
+        return @hls_vtt_uri if instance_variable_defined?(:@hls_vtt_uri)
+
+        @hls_vtt_uri ||= find_hls_vtt_uri
+      end
+
       def mpeg_dash_uri
         return @mpeg_dash_uri if instance_variable_defined?(:@mpeg_dash_uri)
 
@@ -70,6 +76,22 @@ def build_mpeg_dash_uri
         log_invalid_uri(relative_path, e)
       end
 
+      def find_hls_vtt_uri
+        return unless hls_uri_exists?
+        return unless (hls_manifest = do_get(hls_uri, ignore_errors: true))
+
+        playlist = M3u8::Playlist.read(hls_manifest)
+        vtt_manifest = playlist.items.find { |p| p.group_id == 'subs' }
+        hls_vtt_manifest_uri = hls_uri.merge(vtt_manifest.uri)
+
+        return unless (hls_vtt_manifest = do_get(hls_vtt_manifest_uri, ignore_errors: true))
+
+        vtt_playlist = M3u8::Playlist.read(hls_vtt_manifest)
+        return unless (hls_vtt_path_relative = vtt_playlist.items.first.segment)
+
+        hls_uri.merge(hls_vtt_path_relative)
+      end
+
       def find_dash_vtt_uri
         return unless (dash_uri = mpeg_dash_uri)
         return unless (dash_manifest = do_get(dash_uri, ignore_errors: true))
diff --git a/spec/data/breslin1-playlist.m3u8 b/spec/data/breslin1-playlist.m3u8
new file mode 100644
index 0000000..1508454
--- /dev/null
+++ b/spec/data/breslin1-playlist.m3u8
@@ -0,0 +1,5 @@
+#EXTM3U
+#EXT-X-VERSION:3
+#EXT-X-MEDIA:TYPE=SUBTITLES,GROUP-ID="subs",NAME="English",FORCED=NO,AUTOSELECT=YES,URI="subtitlelist_leng_w289808588.m3u8",LANGUAGE="eng"
+#EXT-X-STREAM-INF:BANDWIDTH=294961,CODECS="avc1.4d4028,mp4a.40.2",RESOLUTION=320x212,SUBTITLES="subs"
+chunklist_w289808588.m3u8
diff --git a/spec/data/subtitlelist_leng_w289808588.m3u8 b/spec/data/subtitlelist_leng_w289808588.m3u8
new file mode 100644
index 0000000..db79b18
--- /dev/null
+++ b/spec/data/subtitlelist_leng_w289808588.m3u8
@@ -0,0 +1,7 @@
+#EXTM3U
+#EXT-X-VERSION:3
+#EXT-X-TARGETDURATION:250
+#EXT-X-MEDIA-SEQUENCE:0
+#EXTINF:140.165,
+subtitlechunk_leng_w289808588_0.webvtt
+#EXT-X-ENDLIST
diff --git a/spec/lib/av_player/track_extensions_spec.rb b/spec/lib/av_player/track_extensions_spec.rb
index 81db239..37e36c9 100644
--- a/spec/lib/av_player/track_extensions_spec.rb
+++ b/spec/lib/av_player/track_extensions_spec.rb
@@ -140,6 +140,25 @@ module AV
           expected_uri = URI.join(AV::Config.wowza_base_uri, expected_vtt_path)
           expect(track.dash_vtt_uri).to eq(expected_uri)
         end
+
+        describe :hls_vtt_uri do
+          it 'returns the HLS VTT URI' do
+            track = Track.new(sort_order: 0, path: 'ROHOVideo/breslin1.mp4')
+            hls_uri = track.hls_uri
+
+            expected_manifest_path = '/ROHOVideo/mp4:breslin1.mp4/playlist.m3u8'
+            expect(hls_uri.path).to eq(expected_manifest_path) # just to be sure
+
+            stub_request(:head, hls_uri).to_return(status: 200)
+            stub_request(:get, hls_uri).to_return(body: File.read('spec/data/breslin1-playlist.m3u8'))
+            subtitle_list_uri = hls_uri.merge('subtitlelist_leng_w289808588.m3u8')
+            stub_request(:get, subtitle_list_uri).to_return(body: File.read('spec/data/subtitlelist_leng_w289808588.m3u8'))
+
+            expected_vtt_path = expected_manifest_path.sub(%r{[^/]+$}, 'subtitlechunk_leng_w289808588_0.webvtt')
+            expected_uri = URI.join(AV::Config.wowza_base_uri, expected_vtt_path)
+            expect(track.hls_vtt_uri).to eq(expected_uri)
+          end
+        end
       end
     end
   end