From 51602f4e3faeb7618fc3da1aab700708f357b358 Mon Sep 17 00:00:00 2001 From: Luc Sinet Date: Thu, 28 May 2020 11:25:13 +0200 Subject: [PATCH 01/29] develop #comment General documentation improvements. - Use the read the doc theme. - Add a gendoc hook in the OpenCast.sh script. --- OpenCast.sh | 10 +++++++- docs/source/OpenCast.app.service.rst | 8 +++++++ docs/source/OpenCast.app.workflow.rst | 8 +++++++ docs/source/OpenCast.infra.event.rst | 8 +++++++ docs/source/OpenCast.infra.io.rst | 6 ++--- docs/source/OpenCast.infra.media.rst | 8 ------- docs/source/OpenCast.infra.rst | 12 ++++++++++ docs/source/conf.py | 4 ++-- poetry.lock | 33 +++++++++++++++++++-------- pyproject.toml | 1 + 10 files changed, 75 insertions(+), 23 deletions(-) diff --git a/OpenCast.sh b/OpenCast.sh index 6be841d1..91fc1e08 100755 --- a/OpenCast.sh +++ b/OpenCast.sh @@ -5,6 +5,7 @@ PROJECT_NAME="OpenCast" PROJECT_API_PORT="2020" PROJECT_WEBAPP_PORT="8081" LOG_DIR="log" +DOC_DIR="docs" LOG_FILE="$PROJECT_NAME.log" TEST_DIR="test" @@ -84,6 +85,13 @@ function test() { fi } +function gendoc() { + cd "$DOC_DIR" || exit 1 + + run_in_env make html + xdg-open "build/html/index.html" +} + function run_in_env() { poetry install poetry run "$@" @@ -93,7 +101,7 @@ function run_in_env() { # This is likely to be done by the display manager, but not always (lightdm). source ~/.profile -COMMANDS=("start" "stop" "restart" "update" "status" "logs" "test") +COMMANDS=("start" "stop" "restart" "update" "status" "logs" "test" "gendoc") if element_in "$1" "${COMMANDS[@]}"; then COMMAND="$1" shift diff --git a/docs/source/OpenCast.app.service.rst b/docs/source/OpenCast.app.service.rst index 88470566..5f045264 100644 --- a/docs/source/OpenCast.app.service.rst +++ b/docs/source/OpenCast.app.service.rst @@ -4,6 +4,14 @@ OpenCast.app.service package Submodules ---------- +OpenCast.app.service.error module +--------------------------------- + +.. automodule:: OpenCast.app.service.error + :members: + :undoc-members: + :show-inheritance: + OpenCast.app.service.module module ---------------------------------- diff --git a/docs/source/OpenCast.app.workflow.rst b/docs/source/OpenCast.app.workflow.rst index 10cd05be..3d00bd7e 100644 --- a/docs/source/OpenCast.app.workflow.rst +++ b/docs/source/OpenCast.app.workflow.rst @@ -4,6 +4,14 @@ OpenCast.app.workflow package Submodules ---------- +OpenCast.app.workflow.factory module +------------------------------------ + +.. automodule:: OpenCast.app.workflow.factory + :members: + :undoc-members: + :show-inheritance: + OpenCast.app.workflow.player module ----------------------------------- diff --git a/docs/source/OpenCast.infra.event.rst b/docs/source/OpenCast.infra.event.rst index b2b4506c..1b75745f 100644 --- a/docs/source/OpenCast.infra.event.rst +++ b/docs/source/OpenCast.infra.event.rst @@ -4,6 +4,14 @@ OpenCast.infra.event package Submodules ---------- +OpenCast.infra.event.downloader module +-------------------------------------- + +.. automodule:: OpenCast.infra.event.downloader + :members: + :undoc-members: + :show-inheritance: + OpenCast.infra.event.event module --------------------------------- diff --git a/docs/source/OpenCast.infra.io.rst b/docs/source/OpenCast.infra.io.rst index c6242ed2..e86cfe30 100644 --- a/docs/source/OpenCast.infra.io.rst +++ b/docs/source/OpenCast.infra.io.rst @@ -12,10 +12,10 @@ OpenCast.infra.io.download\_logger module :undoc-members: :show-inheritance: -OpenCast.infra.io.facade module -------------------------------- +OpenCast.infra.io.error module +------------------------------ -.. automodule:: OpenCast.infra.io.facade +.. automodule:: OpenCast.infra.io.error :members: :undoc-members: :show-inheritance: diff --git a/docs/source/OpenCast.infra.media.rst b/docs/source/OpenCast.infra.media.rst index 84879062..d5c75fe6 100644 --- a/docs/source/OpenCast.infra.media.rst +++ b/docs/source/OpenCast.infra.media.rst @@ -12,14 +12,6 @@ OpenCast.infra.media.error module :undoc-members: :show-inheritance: -OpenCast.infra.media.facade module ----------------------------------- - -.. automodule:: OpenCast.infra.media.facade - :members: - :undoc-members: - :show-inheritance: - OpenCast.infra.media.factory module ----------------------------------- diff --git a/docs/source/OpenCast.infra.rst b/docs/source/OpenCast.infra.rst index 2bce195b..fca0f76e 100644 --- a/docs/source/OpenCast.infra.rst +++ b/docs/source/OpenCast.infra.rst @@ -13,6 +13,18 @@ Subpackages OpenCast.infra.log OpenCast.infra.media +Submodules +---------- + +OpenCast.infra.facade module +---------------------------- + +.. automodule:: OpenCast.infra.facade + :members: + :undoc-members: + :show-inheritance: + + Module contents --------------- diff --git a/docs/source/conf.py b/docs/source/conf.py index fa756d80..8f218d8d 100644 --- a/docs/source/conf.py +++ b/docs/source/conf.py @@ -13,7 +13,7 @@ import os import sys -sys.path.insert(0, os.path.abspath("../../")) +sys.path.insert(0, os.path.abspath("../..")) # -- Project information ----------------------------------------------------- @@ -59,7 +59,7 @@ # The theme to use for HTML and HTML Help pages. See the documentation for # a list of builtin themes. # -html_theme = "alabaster" +html_theme = "sphinx_rtd_theme" # Add any paths that contain custom static files (such as style sheets) here, # relative to this directory. They are copied after the builtin static files, diff --git a/poetry.lock b/poetry.lock index b3ccb356..02ecf27a 100644 --- a/poetry.lock +++ b/poetry.lock @@ -324,15 +324,15 @@ category = "dev" description = "Pytest plugin for measuring coverage." name = "pytest-cov" optional = false -python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*" -version = "2.8.1" +python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*, !=3.4.*" +version = "2.9.0" [package.dependencies] coverage = ">=4.4" pytest = ">=3.6" [package.extras] -testing = ["fields", "hunter", "process-tests (2.0.2)", "six", "virtualenv"] +testing = ["fields", "hunter", "process-tests (2.0.2)", "six", "pytest-xdist", "virtualenv"] [[package]] category = "main" @@ -374,7 +374,7 @@ description = "Python 2 and 3 compatibility utilities" name = "six" optional = false python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*" -version = "1.14.0" +version = "1.15.0" [[package]] category = "main" @@ -416,6 +416,17 @@ docs = ["sphinxcontrib-websupport"] lint = ["flake8 (>=3.5.0)", "flake8-import-order", "mypy (>=0.770)", "docutils-stubs"] test = ["pytest", "pytest-cov", "html5lib", "typed-ast", "cython"] +[[package]] +category = "main" +description = "Read the Docs theme for Sphinx" +name = "sphinx-rtd-theme" +optional = false +python-versions = "*" +version = "0.4.3" + +[package.dependencies] +sphinx = "*" + [[package]] category = "main" description = "sphinxcontrib-applehelp is a sphinx extension which outputs Apple help books" @@ -562,7 +573,7 @@ docs = ["sphinx", "jaraco.packaging (>=3.2)", "rst.linker (>=1.9)"] testing = ["jaraco.itertools", "func-timeout"] [metadata] -content-hash = "088668d325e05f65d65e13e4af2ecdb020e35339c0ce9b024708970ba8e7da52" +content-hash = "9b678dad850bc124a93387d7640940365d3f7d35cddd2d637ec9c793aed3af8a" python-versions = "^3.7" [metadata.files] @@ -757,8 +768,8 @@ pytest = [ {file = "pytest-3.10.1.tar.gz", hash = "sha256:e246cf173c01169b9617fc07264b7b1316e78d7a650055235d6d897bc80d9660"}, ] pytest-cov = [ - {file = "pytest-cov-2.8.1.tar.gz", hash = "sha256:cc6742d8bac45070217169f5f72ceee1e0e55b0221f54bcf24845972d3a47f2b"}, - {file = "pytest_cov-2.8.1-py2.py3-none-any.whl", hash = "sha256:cdbdef4f870408ebdbfeb44e63e07eb18bb4619fae852f6e760645fa36172626"}, + {file = "pytest-cov-2.9.0.tar.gz", hash = "sha256:b6a814b8ed6247bd81ff47f038511b57fe1ce7f4cc25b9106f1a4b106f1d9322"}, + {file = "pytest_cov-2.9.0-py2.py3-none-any.whl", hash = "sha256:c87dfd8465d865655a8213859f1b4749b43448b5fae465cb981e16d52a811424"}, ] pytz = [ {file = "pytz-2020.1-py2.py3-none-any.whl", hash = "sha256:a494d53b6d39c3c6e44c3bec237336e14305e4f29bbf800b599253057fbb79ed"}, @@ -782,8 +793,8 @@ requests = [ {file = "requests-2.23.0.tar.gz", hash = "sha256:b3f43d496c6daba4493e7c431722aeb7dbc6288f52a6e04e7b6023b0247817e6"}, ] six = [ - {file = "six-1.14.0-py2.py3-none-any.whl", hash = "sha256:8f3cd2e254d8f793e7f3d6d9df77b92252b52637291d0f0da013c76ea2724b6c"}, - {file = "six-1.14.0.tar.gz", hash = "sha256:236bdbdce46e6e6a3d61a337c0f8b763ca1e8717c03b369e87a7ec7ce1319c0a"}, + {file = "six-1.15.0-py2.py3-none-any.whl", hash = "sha256:8b74bedcbbbaca38ff6d7491d76f2b06b3592611af620f8426e82dddb04a5ced"}, + {file = "six-1.15.0.tar.gz", hash = "sha256:30639c035cdb23534cd4aa2dd52c3bf48f06e5f4a941509c8bafd8ce11080259"}, ] snowballstemmer = [ {file = "snowballstemmer-2.0.0-py2.py3-none-any.whl", hash = "sha256:209f257d7533fdb3cb73bdbd24f436239ca3b2fa67d56f6ff88e86be08cc5ef0"}, @@ -793,6 +804,10 @@ sphinx = [ {file = "Sphinx-3.0.3-py3-none-any.whl", hash = "sha256:f5505d74cf9592f3b997380f9bdb2d2d0320ed74dd69691e3ee0644b956b8d83"}, {file = "Sphinx-3.0.3.tar.gz", hash = "sha256:62edfd92d955b868d6c124c0942eba966d54b5f3dcb4ded39e65f74abac3f572"}, ] +sphinx-rtd-theme = [ + {file = "sphinx_rtd_theme-0.4.3-py2.py3-none-any.whl", hash = "sha256:00cf895504a7895ee433807c62094cf1e95f065843bf3acd17037c3e9a2becd4"}, + {file = "sphinx_rtd_theme-0.4.3.tar.gz", hash = "sha256:728607e34d60456d736cc7991fd236afb828b21b82f956c5ea75f94c8414040a"}, +] sphinxcontrib-applehelp = [ {file = "sphinxcontrib-applehelp-1.0.2.tar.gz", hash = "sha256:a072735ec80e7675e3f432fcae8610ecf509c5f1869d17e2eecff44389cdbc58"}, {file = "sphinxcontrib_applehelp-1.0.2-py2.py3-none-any.whl", hash = "sha256:806111e5e962be97c29ec4c1e7fe277bfd19e9652fb1a4392105b43e01af885a"}, diff --git a/pyproject.toml b/pyproject.toml index d284bd21..0d1d20c4 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -26,6 +26,7 @@ transitions = "^0.8.1" structlog = "^20.1.0" colorama = "^0.4.3" sphinx = "^3.0.3" +sphinx_rtd_theme = "^0.4.3" [tool.poetry.dev-dependencies] pytest = "^3.0" From 0811e0f80a6a728c700e9e7e030dd69da1938d18 Mon Sep 17 00:00:00 2001 From: Luc Sinet Date: Sun, 31 May 2020 18:09:55 +0200 Subject: [PATCH 02/29] fetch-subtitles-online #comment Change language format to match the one of ydl. --- OpenCast/config.py | 2 +- config.yml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/OpenCast/config.py b/OpenCast/config.py index e8bf5093..50597a57 100644 --- a/OpenCast/config.py +++ b/OpenCast/config.py @@ -179,7 +179,7 @@ def _override_from_env(self, content: dict, env: dict, key: list): "max_concurrency": 3 }, "subtitle": { - "language": "eng" + "language": "en" } }, check_env=True) # fmt: on diff --git a/config.yml b/config.yml index 190f2dcf..0c3bd121 100644 --- a/config.yml +++ b/config.yml @@ -24,4 +24,4 @@ downloader: subtitle: # The default language for subtitles. - language: eng + language: en From 103c348e1420395dca154dd5423620944a89c2b7 Mon Sep 17 00:00:00 2001 From: Luc Sinet Date: Sun, 31 May 2020 18:17:35 +0200 Subject: [PATCH 03/29] fetch-subtitles-online #comment Refactor the video_downloader method names. --- OpenCast/app/controller/player_monitor.py | 2 +- OpenCast/app/service/video.py | 6 +++--- OpenCast/domain/service/source.py | 8 ++++---- .../infra/io/{video_downloader.py => downloader.py} | 12 ++++++------ OpenCast/infra/io/factory.py | 6 +++--- test/integration/app/service/test_video.py | 8 ++++---- 6 files changed, 21 insertions(+), 21 deletions(-) rename OpenCast/infra/io/{video_downloader.py => downloader.py} (90%) diff --git a/OpenCast/app/controller/player_monitor.py b/OpenCast/app/controller/player_monitor.py index 82c6fe06..c2b2cdad 100755 --- a/OpenCast/app/controller/player_monitor.py +++ b/OpenCast/app/controller/player_monitor.py @@ -20,7 +20,7 @@ def __init__(self, app_facade, infra_facade, data_facade, service_factory): super(PlayerMonitController, self).__init__(app_facade) self._source_service = service_factory.make_source_service( - infra_facade.io_factory.make_video_downloader(app_facade.evt_dispatcher) + infra_facade.io_factory.make_downloader(app_facade.evt_dispatcher) ) self._player_repo = data_facade.player_repo self._video_repo = data_facade.video_repo diff --git a/OpenCast/app/service/video.py b/OpenCast/app/service/video.py index ef239e04..175c2219 100644 --- a/OpenCast/app/service/video.py +++ b/OpenCast/app/service/video.py @@ -13,7 +13,7 @@ def __init__(self, app_facade, service_factory, data_facade, io_factory): logger = structlog.get_logger(__name__) super(VideoService, self).__init__(app_facade, logger, self, video_cmds) self._video_repo = data_facade.video_repo - self._downloader = io_factory.make_video_downloader(app_facade.evt_dispatcher) + self._downloader = io_factory.make_downloader(app_facade.evt_dispatcher) self._source_service = service_factory.make_source_service(self._downloader) self._subtitle_service = service_factory.make_subtitle_service( io_factory.make_ffmpeg_wrapper() @@ -41,7 +41,7 @@ def impl(ctx, video, metadata): ctx.update(video) video = self._video_repo.get(cmd.model_id) - metadata = self._source_service.fetch_metadata(video) + metadata = self._source_service.pick_stream_metadata(video) if metadata is None: self._abort_operation(cmd, "can't fetch metadata") return @@ -73,7 +73,7 @@ def abort_operation(evt): {DownloadSuccess: video_downloaded, DownloadError: abort_operation}, times=1, ) - self._downloader.download(cmd.id, video) + self._downloader.download_video(cmd.id, video) def _fetch_video_subtitle(self, cmd): def impl(ctx): diff --git a/OpenCast/domain/service/source.py b/OpenCast/domain/service/source.py index 9f9764ba..3cb88365 100644 --- a/OpenCast/domain/service/source.py +++ b/OpenCast/domain/service/source.py @@ -2,8 +2,8 @@ class SourceService: - def __init__(self, video_downloader): - self._downloader = video_downloader + def __init__(self, downloader): + self._downloader = downloader def is_playlist(self, source): return "/playlist" in source @@ -11,7 +11,7 @@ def is_playlist(self, source): def unfold(self, source): return self._downloader.unfold_playlist(source) - def fetch_metadata(self, video): + def pick_stream_metadata(self, video): if video.is_file(): return {"title": str(Path(video.source).name)} - return self._downloader.fetch_metadata(video.source, ["title"]) + return self._downloader.pick_stream_metadata(video.source, ["title"]) diff --git a/OpenCast/infra/io/video_downloader.py b/OpenCast/infra/io/downloader.py similarity index 90% rename from OpenCast/infra/io/video_downloader.py rename to OpenCast/infra/io/downloader.py index 14d51d68..d25e4b73 100644 --- a/OpenCast/infra/io/video_downloader.py +++ b/OpenCast/infra/io/downloader.py @@ -6,7 +6,7 @@ from .download_logger import DownloadLogger -class VideoDownloader: +class Downloader: def __init__(self, executor, evt_dispatcher): self._executor = executor self._evt_dispatcher = evt_dispatcher @@ -14,7 +14,7 @@ def __init__(self, executor, evt_dispatcher): self._dl_logger = DownloadLogger(self._logger) self._log_debug = False # self._dl_logger.is_enabled_for(logging.DEBUG) - def download(self, op_id, video): + def download_video(self, op_id, video): def impl(): self._logger.debug("Downloading", video=video) options = { @@ -42,11 +42,11 @@ def impl(): self._logger.debug("Queing", video=video) self._executor.submit(impl) - def fetch_metadata(self, url, fields): + def pick_stream_metadata(self, url, fields): options = { "noplaylist": True, } - data = self._fetch_metadata(url, options) + data = self._download_stream_metadata(url, options) if data is None: return None return {k: data[k] for k in fields} @@ -56,7 +56,7 @@ def unfold_playlist(self, url): "extract_flat": "in_playlist", } # Download the playlist data without downloading the videos. - data = self._fetch_metadata(url, options) + data = self._download_stream_metadata(url, options) if data is None: return [] @@ -65,7 +65,7 @@ def unfold_playlist(self, url): urls = [base_url + "/watch?v=" + entry["id"] for entry in data["entries"]] return urls - def _fetch_metadata(self, url, options): + def _download_stream_metadata(self, url, options): self._logger.debug("Fetching metadata", url=url) options.update( { diff --git a/OpenCast/infra/io/factory.py b/OpenCast/infra/io/factory.py index bf3737e6..2b0d1cd2 100644 --- a/OpenCast/infra/io/factory.py +++ b/OpenCast/infra/io/factory.py @@ -1,6 +1,6 @@ from .ffmpeg_wrapper import FFmpegWrapper from .server import Server -from .video_downloader import VideoDownloader +from .downloader import Downloader class IoFactory: @@ -13,5 +13,5 @@ def make_ffmpeg_wrapper(self, *args): def make_server(self, *args): return Server(*args) - def make_video_downloader(self, *args): - return VideoDownloader(self._downloader_executor, *args) + def make_downloader(self, *args): + return Downloader(self._downloader_executor, *args) diff --git a/test/integration/app/service/test_video.py b/test/integration/app/service/test_video.py index 4f8b13a4..2d427017 100644 --- a/test/integration/app/service/test_video.py +++ b/test/integration/app/service/test_video.py @@ -20,7 +20,7 @@ def setUp(self): self.downloader = Mock() self.ffmpeg_wrapper = Mock() self.io_factory = self.infra_facade.io_factory - self.io_factory.make_video_downloader.return_value = self.downloader + self.io_factory.make_downloader.return_value = self.downloader self.io_factory.make_ffmpeg_wrapper.return_value = self.ffmpeg_wrapper self.service = VideoService( @@ -56,7 +56,7 @@ def test_identify_video(self): self.data_producer.video("source", None).populate(self.data_facade) title = "video_title" - self.downloader.fetch_metadata.return_value = {"title": title} + self.downloader.pick_stream_metadata.return_value = {"title": title} video_id = IdentityService.id_video("source") self.evt_expecter.expect(Evt.VideoIdentified, title).from_( @@ -73,7 +73,7 @@ def test_retrieve_video_success(self): def dispatch_downloaded(op_id, *args): self.app_facade.evt_dispatcher.dispatch(DownloadSuccess(op_id)) - self.downloader.download.side_effect = dispatch_downloaded + self.downloader.download_video.side_effect = dispatch_downloaded output_dir = config["downloader.output_directory"] path = Path(output_dir) / f"{video_title}.mp4" self.evt_expecter.expect(Evt.VideoRetrieved, path).from_( @@ -92,7 +92,7 @@ def dispatch_error(op_id, *args): DownloadError(op_id, "Download error") ) - self.downloader.download.side_effect = dispatch_error + self.downloader.download_video.side_effect = dispatch_error output_dir = config["downloader.output_directory"] self.evt_expecter.expect(OperationError, "Download error").from_( Cmd.RetrieveVideo, video_id, output_dir From 7d612d437435658e9d762193d376126e402b562d Mon Sep 17 00:00:00 2001 From: Luc Sinet Date: Mon, 1 Jun 2020 14:29:58 +0200 Subject: [PATCH 04/29] fetch-subtitles-online #comment Add the subtitle download capability to the downloader. - Update the subtitle service to find subtitles from multiple sources. --- OpenCast/domain/service/subtitle.py | 46 +++++++++++++++++++++++----- OpenCast/infra/io/download_logger.py | 2 +- OpenCast/infra/io/downloader.py | 46 ++++++++++++++++++++++------ 3 files changed, 76 insertions(+), 18 deletions(-) diff --git a/OpenCast/domain/service/subtitle.py b/OpenCast/domain/service/subtitle.py index 2f52b0da..c27376a1 100644 --- a/OpenCast/domain/service/subtitle.py +++ b/OpenCast/domain/service/subtitle.py @@ -4,14 +4,32 @@ class SubtitleService: - def __init__(self, ffmpeg_wrapper): + def __init__(self, subtitle_converter, ffmpeg_wrapper, downloader): self._logger = structlog.get_logger(__name__) + self._subtitle_converter = subtitle_converter self._ffmpeg_wrapper = ffmpeg_wrapper + self._downloader = downloader - def load_from_disk(self, video, language): - video_name = video.path.name.rsplit(".", 1)[0] - parent_path = video.path.parents[0] - subtitle = f"{parent_path}/{video_name}.srt" + def fetch_subtitle( + self, video, language: str, search_source=True, search_online=True + ) -> Path: + subtitle = self._load_from_disk(video.path, language) + if subtitle is not None: + return subtitle + + if search_source: + subtitle = self._download_from_source(video.source, video.path, language) + if subtitle is not None: + return subtitle + + if search_online: + pass # TODO + + return None + + def _load_from_disk(self, video_path: Path, language: str) -> str: + parent_path = video_path.parents[0] + subtitle = str(video_path.with_suffix(".srt")) # Find the matching subtitle from a .srt file srtFiles = list(parent_path.glob("*.srt")) @@ -21,8 +39,8 @@ def load_from_disk(self, video, language): # Extract file metadata # Find subtitle with matching language - self._logger.debug("Searching softcoded subtitles", video=video) - metadata = self._ffmpeg_wrapper.probe(video.path) + self._logger.debug("Searching softcoded subtitles", subtitle=subtitle) + metadata = self._ffmpeg_wrapper.probe(video_path) for stream in metadata["streams"]: self._logger.debug( f"Channel #{stream['index']}", @@ -35,10 +53,22 @@ def load_from_disk(self, video, language): ): self._logger.debug(f"Match: {subtitle}") if self._ffmpeg_wrapper.extract_stream( - src=video.path, + src=video_path, dest=subtitle, stream_idx=stream["index"], override=False, ): return subtitle return None + + def _download_from_source( + self, video_source: str, video_path: Path, language: str + ) -> str: + dest = str(video_path.with_suffix("")) + subtitle = self._downloader.download_subtitle( + video_source, dest, language, ["vtt"] + ) + if subtitle is None or Path(subtitle).suffix == "srt": + return subtitle + + return self._subtitle_converter.vtt_to_srt(Path(subtitle)) diff --git a/OpenCast/infra/io/download_logger.py b/OpenCast/infra/io/download_logger.py index d75e013d..21643e65 100644 --- a/OpenCast/infra/io/download_logger.py +++ b/OpenCast/infra/io/download_logger.py @@ -9,7 +9,7 @@ def is_enabled_for(self, level): return False # return self._logger.isEnabledFor(level) - def log_progress(self, d): + def log_download_progress(self, d): status = d.get("status", "N/A") if status not in ["downloading", "error", "finished"]: return diff --git a/OpenCast/infra/io/downloader.py b/OpenCast/infra/io/downloader.py index d25e4b73..cf84e9d8 100644 --- a/OpenCast/infra/io/downloader.py +++ b/OpenCast/infra/io/downloader.py @@ -1,3 +1,5 @@ +from typing import List + import youtube_dl import structlog @@ -14,34 +16,60 @@ def __init__(self, executor, evt_dispatcher): self._dl_logger = DownloadLogger(self._logger) self._log_debug = False # self._dl_logger.is_enabled_for(logging.DEBUG) - def download_video(self, op_id, video): + def download_video(self, op_id, source: str, dest: str): def impl(): - self._logger.debug("Downloading", video=video) + self._logger.debug("Downloading", video=dest) options = { "format": "bestvideo[ext=mp4]+bestaudio[ext=m4a]/" "bestvideo+bestaudio/best", "debug_printtraffic": self._log_debug, "noplaylist": True, "merge_output_format": "mp4", - "outtmpl": str(video.path), + "outtmpl": dest, "quiet": True, - "progress_hooks": [self._dl_logger.log_progress], + "progress_hooks": [self._dl_logger.log_download_progress], } ydl = youtube_dl.YoutubeDL(options) with ydl: # Download the video try: - ydl.download([video.source]) + ydl.download([source]) except Exception as e: - self._logger.error("Download error", video=video, error=e) + self._logger.error( + "Download error", video=dest, source=source, error=e + ) self._evt_dispatcher.dispatch(DownloadError(op_id, str(e))) return - self._logger.debug("Download success", video=video) + self._logger.debug("Download success", video=dest) self._evt_dispatcher.dispatch(DownloadSuccess(op_id)) - self._logger.debug("Queing", video=video) + self._logger.debug("Queing", video=dest) self._executor.submit(impl) + def download_subtitle(self, url: str, dest: str, lang: str, exts: List[str]): + self._logger.debug("Downloading subtitle", subtitle=dest, lang=lang) + + for ext in exts: + options = { + "skip_download": True, + "subtitleslangs": [lang], + "subtitlesformat": ext, + "writeautomaticsub": True, + "outtmpl": dest, + "progress_hooks": [self._dl_logger.log_download_progress], + "quiet": True, + } + ydl = youtube_dl.YoutubeDL(options) + with ydl: + try: + ydl.download([url]) + return f"{dest}.{lang}.{ext}" + except Exception as e: + self._logger.error( + "Subtitle download error", subtitle=dest, ext=ext, error=e + ) + return None + def pick_stream_metadata(self, url, fields): options = { "noplaylist": True, @@ -72,7 +100,7 @@ def _download_stream_metadata(self, url, options): "ignoreerrors": True, # Causes ydl to return None on error "debug_printtraffic": self._log_debug, "quiet": True, - "progress_hooks": [self._dl_logger.log_progress], + "progress_hooks": [self._dl_logger.log_download_progress], } ) ydl = youtube_dl.YoutubeDL(options) From d3125b618f1953ebd3c2d1e51ff57afeb13b650c Mon Sep 17 00:00:00 2001 From: Luc Sinet Date: Mon, 1 Jun 2020 14:35:00 +0200 Subject: [PATCH 05/29] fetch-subtitles-online #comment Add infra service for converting subtitles from vtt to srt. --- OpenCast/__init__.py | 4 ++- OpenCast/app/service/video.py | 6 ++-- OpenCast/domain/service/factory.py | 7 ++++- OpenCast/infra/service/__init__.py | 0 OpenCast/infra/service/factory.py | 6 ++++ OpenCast/infra/service/subtitle.py | 34 +++++++++++++++++++++++ test/integration/app/service/util.py | 3 +- test/unit/domain/service/test_subtitle.py | 2 +- 8 files changed, 55 insertions(+), 7 deletions(-) create mode 100644 OpenCast/infra/service/__init__.py create mode 100644 OpenCast/infra/service/factory.py create mode 100644 OpenCast/infra/service/subtitle.py diff --git a/OpenCast/__init__.py b/OpenCast/__init__.py index 18ad37b5..febb60ae 100644 --- a/OpenCast/__init__.py +++ b/OpenCast/__init__.py @@ -15,6 +15,7 @@ from .infra.io.factory import IoFactory from .infra.log.module import init as init_logging from .infra.media.factory import MediaFactory +from .infra.service.factory import ServiceFactory as InfraServiceFactory def main(argv=None): @@ -35,7 +36,8 @@ def main(argv=None): app_executor = ThreadPoolExecutor(max_workers=1) app_facade = AppFacade(app_executor) - service_factory = ServiceFactory() + infra_service_factory = InfraServiceFactory() + service_factory = ServiceFactory(infra_service_factory) repo_factory = RepoFactory() data_facade = DataFacade(repo_factory) diff --git a/OpenCast/app/service/video.py b/OpenCast/app/service/video.py index 175c2219..e7995047 100644 --- a/OpenCast/app/service/video.py +++ b/OpenCast/app/service/video.py @@ -16,7 +16,7 @@ def __init__(self, app_facade, service_factory, data_facade, io_factory): self._downloader = io_factory.make_downloader(app_facade.evt_dispatcher) self._source_service = service_factory.make_source_service(self._downloader) self._subtitle_service = service_factory.make_subtitle_service( - io_factory.make_ffmpeg_wrapper() + io_factory.make_ffmpeg_wrapper(), self._downloader ) # Command handler interface implementation @@ -73,12 +73,12 @@ def abort_operation(evt): {DownloadSuccess: video_downloaded, DownloadError: abort_operation}, times=1, ) - self._downloader.download_video(cmd.id, video) + self._downloader.download_video(cmd.id, video.source, str(video.path)) def _fetch_video_subtitle(self, cmd): def impl(ctx): video = self._video_repo.get(cmd.model_id) - video.subtitle = self._subtitle_service.load_from_disk(video, cmd.language) + video.subtitle = self._subtitle_service.fetch_subtitle(video, cmd.language) ctx.update(video) self._start_transaction(self._video_repo, cmd.id, impl) diff --git a/OpenCast/domain/service/factory.py b/OpenCast/domain/service/factory.py index a5adc13c..a04c2226 100644 --- a/OpenCast/domain/service/factory.py +++ b/OpenCast/domain/service/factory.py @@ -3,8 +3,13 @@ class ServiceFactory: + def __init__(self, infra_service_factory): + self._infra_service_factory = infra_service_factory + def make_source_service(self, *args): return SourceService(*args) def make_subtitle_service(self, *args): - return SubtitleService(*args) + return SubtitleService( + self._infra_service_factory.make_subtitle_converter(), *args + ) diff --git a/OpenCast/infra/service/__init__.py b/OpenCast/infra/service/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/OpenCast/infra/service/factory.py b/OpenCast/infra/service/factory.py new file mode 100644 index 00000000..c6e08fe6 --- /dev/null +++ b/OpenCast/infra/service/factory.py @@ -0,0 +1,6 @@ +from .subtitle import SubtitleConverter + + +class ServiceFactory: + def make_subtitle_converter(self, *args): + return SubtitleConverter(*args) diff --git a/OpenCast/infra/service/subtitle.py b/OpenCast/infra/service/subtitle.py new file mode 100644 index 00000000..b8177d1e --- /dev/null +++ b/OpenCast/infra/service/subtitle.py @@ -0,0 +1,34 @@ +from pathlib import Path + + +class SubtitleConverter: + def vtt_to_srt(self, subtitle: Path): + print(f"subtitle: {subtitle}") + dest = subtitle.with_suffix(".srt") + with open(subtitle, "r") as fp: + content = fp.readlines() + + output = [] + subtitle_count = 0 + in_caption = False + for line in content: + if "-->" in line: + tokens = line.split(" --> ") + if len(tokens) < 3: + continue + + in_caption = True + if subtitle_count > 0: + output.append("\n") + subtitle_count += 1 + output.append(f"{subtitle_count}\n") + output.append(line.replace(".", ",")) + elif line != "" and in_caption: # Skip vtt header and notes + output.append(line) + else: + in_caption = False + + with open(dest, "w") as fp: + fp.writelines(output) + + return dest diff --git a/test/integration/app/service/util.py b/test/integration/app/service/util.py index e455e1ee..a7944393 100644 --- a/test/integration/app/service/util.py +++ b/test/integration/app/service/util.py @@ -22,7 +22,8 @@ def execute_handler(handler, *args): repo_factory = RepoFactory() self.data_facade = DataFacade(repo_factory) - self.service_factory = ServiceFactory() + infraServiceFactory = Mock() + self.service_factory = ServiceFactory(infraServiceFactory) self.infra_facade = InfraFacadeMock() self.evt_expecter = EventExpecter( diff --git a/test/unit/domain/service/test_subtitle.py b/test/unit/domain/service/test_subtitle.py index 31685a1d..b13d2489 100644 --- a/test/unit/domain/service/test_subtitle.py +++ b/test/unit/domain/service/test_subtitle.py @@ -5,7 +5,7 @@ class SubtitleServiceTest(TestCase): def setUp(self): - self._service = SubtitleService(None) + self._service = SubtitleService(None, None, None) def test_load_from_disk(self): pass From 22f1ba759e81e0477bdacfacb0b5cda58cc24ce8 Mon Sep 17 00:00:00 2001 From: Luc Sinet Date: Mon, 1 Jun 2020 14:35:44 +0200 Subject: [PATCH 06/29] fetch-subtitles-online #comment Log the trace of the exceptions caught by the command handlers. --- OpenCast/app/service/service.py | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/OpenCast/app/service/service.py b/OpenCast/app/service/service.py index b8a5c31c..8373d5a0 100644 --- a/OpenCast/app/service/service.py +++ b/OpenCast/app/service/service.py @@ -1,4 +1,5 @@ import inspect +import traceback from functools import partial from OpenCast.infra.data.repo.error import RepoError @@ -40,7 +41,12 @@ def _dispatch_to_handler(self, cmd): self._logger.error("Repo error", cmd=cmd, error=e) retry_count -= 1 except Exception as e: - self._logger.error("Operation error", cmd=cmd, error=e) + self._logger.error( + "Operation error", + cmd=cmd, + error=e, + traceback=traceback.format_exc(), + ) self._abort_operation(cmd, str(e)) return From 4d0980b6089bf021ae6d006fb92860d917b029f5 Mon Sep 17 00:00:00 2001 From: Luc Sinet Date: Wed, 3 Jun 2020 18:58:49 +0200 Subject: [PATCH 07/29] replace-omxplayer #comment Fix sound feature. - Move hardcoded value into constants. --- OpenCast/app/controller/player_monitor.py | 12 ++++++------ OpenCast/domain/model/player.py | 2 ++ 2 files changed, 8 insertions(+), 6 deletions(-) diff --git a/OpenCast/app/controller/player_monitor.py b/OpenCast/app/controller/player_monitor.py index c2b2cdad..36f8ea8a 100755 --- a/OpenCast/app/controller/player_monitor.py +++ b/OpenCast/app/controller/player_monitor.py @@ -82,13 +82,13 @@ def _video(self): elif control == "stop": self._dispatch(Cmd.StopVideo) elif control == "right": - self._dispatch(Cmd.SeekVideo, 30) + self._dispatch(Cmd.SeekVideo, Player.SHORT_TIME_STEP) elif control == "left": - self._dispatch(Cmd.SeekVideo, -30) + self._dispatch(Cmd.SeekVideo, -Player.SHORT_TIME_STEP) elif control == "longright": - self._dispatch(Cmd.SeekVideo, 300) + self._dispatch(Cmd.SeekVideo, Player.LONG_TIME_STEP) elif control == "longleft": - self._dispatch(Cmd.SeekVideo, -300) + self._dispatch(Cmd.SeekVideo, -Player.LONG_TIME_STEP) elif control == "prev": self._dispatch(Cmd.PrevVideo) elif control == "next": @@ -100,9 +100,9 @@ def _video(self): def _sound(self): if request.query["vol"] == "more": - self._dispatcher(Cmd.ChangeVolume, Player.VOLUME_STEP) + self._dispatch(Cmd.ChangeVolume, Player.VOLUME_STEP) else: - self._dispatcher(Cmd.ChangeVolume, -Player.VOLUME_STEP) + self._dispatch(Cmd.ChangeVolume, -Player.VOLUME_STEP) return "1" def _subtitle(self): diff --git a/OpenCast/domain/model/player.py b/OpenCast/domain/model/player.py index 6820fc40..23cf1922 100644 --- a/OpenCast/domain/model/player.py +++ b/OpenCast/domain/model/player.py @@ -10,6 +10,8 @@ class Player(Entity): VOLUME_STEP = 10 SUBTITLE_DELAY_STEP = 100 + SHORT_TIME_STEP = 1000 + LONG_TIME_STEP = 30000 def __init__(self, id_): super(Player, self).__init__(id_) From 91cef19de3cb8d9a53b0a07f4f13435a0808aca2 Mon Sep 17 00:00:00 2001 From: Luc Sinet Date: Wed, 3 Jun 2020 19:03:07 +0200 Subject: [PATCH 08/29] replace-omxplayer #comment Transition from Omxplayer to vlc. - Now than vlc handles h264 hardware decoding and that hevc decoding is on the way it is time to say goodbye to omxplayer. --- OpenCast.sh | 1 - OpenCast/app/service/player.py | 62 ++++----- OpenCast/infra/event/player.py | 2 +- OpenCast/infra/media/factory.py | 9 +- OpenCast/infra/media/player_wrapper.py | 144 ++++++-------------- pyproject.toml | 3 +- setup.sh | 2 +- test/integration/app/service/test_player.py | 15 +- test/shared/infra/media/player_mock.py | 12 -- 9 files changed, 74 insertions(+), 176 deletions(-) delete mode 100644 test/shared/infra/media/player_mock.py diff --git a/OpenCast.sh b/OpenCast.sh index 91fc1e08..c923248e 100755 --- a/OpenCast.sh +++ b/OpenCast.sh @@ -53,7 +53,6 @@ function stop() { # Todo hardcoded port lsof -t -i :2020 | xargs kill >/dev/null 2>&1 lsof -t -i :8081 | xargs kill >/dev/null 2>&1 - sudo killall omxplayer.bin >/dev/null 2>&1 echo "Done." } diff --git a/OpenCast/app/service/player.py b/OpenCast/app/service/player.py index fccb6372..1d90fdc8 100644 --- a/OpenCast/app/service/player.py +++ b/OpenCast/app/service/player.py @@ -1,11 +1,7 @@ -from functools import partial - import structlog from OpenCast.app.command import player as player_cmds from OpenCast.app.error import CommandFailure -from OpenCast.domain.event import player as player_events from OpenCast.domain.model.player_state import PlayerState -from OpenCast.infra.event import player as infra_events from .service import Service @@ -13,9 +9,7 @@ class PlayerService(Service): def __init__(self, app_facade, data_facade, media_factory): logger = structlog.get_logger(__name__) - super(PlayerService, self).__init__( - app_facade, logger, self, player_cmds, infra_events - ) + super(PlayerService, self).__init__(app_facade, logger, self, player_cmds) self._player_repo = data_facade.player_repo self._video_repo = data_facade.video_repo @@ -42,23 +36,20 @@ def play_next(model): # Command handler interface implementation def _play_video(self, cmd): - def queue_video(model, video): - model.queue(video, with_priority=True) - video = self._video_repo.get(cmd.video_id) - self._update(cmd.id, queue_video, video) + self._queue_video_impl(cmd.id, video) self._play_video_impl(cmd.id, video) def _queue_video(self, cmd): - def queue_video(model): - video = self._video_repo.get(cmd.video_id) - model.queue(video) - - self._update(cmd.id, queue_video) + video = self._video_repo.get(cmd.video_id) + self._queue_video_impl(cmd.id, video) def _stop_video(self, cmd): - self._player.stop(cmd.id) - # Model updates are done from the infra event handler + def stop_video(model): + model.stop() + self._player.stop() + + self._update(cmd.id, stop_video) def _toggle_video_state(self, cmd): def pause(model): @@ -101,45 +92,40 @@ def _prev_video(self, cmd): self._play_video_impl(cmd.id, prev_video) def _toggle_subtitle(self, cmd): - def impl(model, state): - model.subtitle_state = state + def impl(model): + model.subtitle_state = not model.subtitle_state - model = self._player_model() - state = not model.subtitle_state - self._player.update_subtitle_state(state) - self._update(cmd.id, impl, state) + self._player.toggle_subtitle() + self._update(cmd.id, impl) def _increase_subtitle_delay(self, cmd): def impl(model): model.subtitle_delay = model.subtitle_delay + cmd.amount + self._player.set_subtitle_delay(model.subtitle_delay) - self._player.increase_subtitle_delay() self._update(cmd.id, impl) def _decrease_subtitle_delay(self, cmd): def impl(model): model.subtitle_delay = model.subtitle_delay - cmd.amount + self._player.set_subtitle_delay(model.subtitle_delay) - self._player.increase_subtitle_delay() self._update(cmd.id, impl) # Private def _play_video_impl(self, cmd_id, video): - def play_video(model, video, *_): - def impl(model): - model.play(video) + def play_video(model): + model.play(video) - self._player.play(video, model.volume) - self._update(cmd_id, impl) + self._player.play(video.id, str(video.path)) + self._update(cmd_id, play_video) - model = self._player_model() - if model.state is not PlayerState.STOPPED: - callback = partial(play_video, model, video) - self._evt_dispatcher.once(player_events.PlayerStopped, callback) - self._player.stop(cmd_id) - else: - play_video(model, video) + def _queue_video_impl(self, cmd_id, video): + def queue_video(model): + model.queue(video) + + self._update(cmd_id, queue_video) def _player_model(self): return self._player_repo.get_player() diff --git a/OpenCast/infra/event/player.py b/OpenCast/infra/event/player.py index d6111449..f4440ad5 100644 --- a/OpenCast/infra/event/player.py +++ b/OpenCast/infra/event/player.py @@ -4,5 +4,5 @@ @dataclass -class PlayerStopped(Event): +class MediaEndReached(Event): pass diff --git a/OpenCast/infra/media/factory.py b/OpenCast/infra/media/factory.py index e6a99fce..8d019a1e 100644 --- a/OpenCast/infra/media/factory.py +++ b/OpenCast/infra/media/factory.py @@ -1,15 +1,8 @@ from pathlib import Path -from omxplayer.player import OMXPlayer - from .player_wrapper import PlayerWrapper class MediaFactory: def make_player(self, *args): - def player_factory(path, command, dbus_name, exit_callback): - player = OMXPlayer(Path(path), command, dbus_name=dbus_name) - player.exitEvent += exit_callback - return player - - return PlayerWrapper(player_factory, *args) + return PlayerWrapper(*args) diff --git a/OpenCast/infra/media/player_wrapper.py b/OpenCast/infra/media/player_wrapper.py index 1034c915..59861931 100644 --- a/OpenCast/infra/media/player_wrapper.py +++ b/OpenCast/infra/media/player_wrapper.py @@ -1,130 +1,64 @@ from threading import Lock - -import psutil +from uuid import UUID import OpenCast.infra.event.player as e import structlog -from omxplayer import keys -from OpenCast.config import config - -from .error import PlayerError +import vlc -# OmxPlayer documentation: https://elinux.org/Omxplayer class PlayerWrapper: - def __init__(self, player_factory, evt_dispatcher): + def __init__(self, evt_dispatcher): self._logger = structlog.get_logger(__name__) - self._player_factory = player_factory self._evt_dispatcher = evt_dispatcher - self._player = None - self._player_lock = Lock() - self._stop_operation_id = None - - def play(self, video, volume): - command = ["--vol", self._downscale(volume)] - if config["player.hide_background"] is True: - command += ["--blank"] - - if video.subtitle is not None: - command += ["--subtitles", video.subtitle] - def start_player(): - self._logger.debug("Opening video", video=video, opt=command) - try: - self._player = self._player_factory( - video.path, - command, - "org.mpris.MediaPlayer2.omxplayer1", - self._on_exit, - ) - return True - except SystemError: - self._logger.error("Dbus error", error="Couldn't connect") - # Kill instance if it is a dbus problem - for proc in psutil.process_iter(): - if "omxplayer" in proc.name(): - self._logger.debug(f"Killing process", process=proc.name()) - proc.kill() - return False + self._instance = vlc.Instance() + self._player = self._instance.media_player_new() + # self._list_player = self._instance.media_list_player_new() + # self._list_player.set_media_player(self._player) + # self._playlist = self._instance.media_list_new() + # self._list_player.set_media_list(self._playlist) + self._id_to_media = {} - player_started = False - with self._player_lock: - for _ in range(5): - player_started = start_player() - if player_started: - break + self._lock = Lock() + self._stop_operation_id = None - if not player_started: - raise PlayerError("error starting the player") + player_events = self._player.event_manager() + player_events.event_attach( + vlc.EventType.MediaPlayerEndReached, self._on_media_end + ) - def stop(self, op_id): - def impl(): - self._stop_operation_id = op_id - self._player.stop() - # Event is dispatched from _on_exit + def play(self, video_id: UUID, video_path: str): + media = self._id_to_media.get(video_id, None) + if media is None: + media = self._instance.media_new(video_path) + self._id_to_media[video_id] = media + print(f"Play video {video_id}") + self._player.set_media(media) + self._player.play() - self._exec_command(impl) + def stop(self): + self._player.stop() def pause(self): - def impl(): - self._player.play_pause() - - self._exec_command(impl) + self._player.pause() def unpause(self): - def impl(): - self._player.play_pause() - - self._exec_command(impl) - - def update_subtitle_state(self, state): - def impl(): - if state is True: - self._player.show_subtitles() - else: - self._player.hide_subtitles() + self._player.pause() - self._exec_command(impl) + def toggle_subtitle(self): + self._player.toggle_teletext() - def increase_subtitle_delay(self): - def impl(): - self._player.action(keys.INCREASE_SUBTITLE_DELAY) - - self._exec_command(impl) - - def decrease_subtitle_delay(self): - def impl(): - self._player.action(keys.DECREASE_SUBTITLE_DELAY) - - self._exec_command(impl) + def set_subtitle_delay(self, delay: int): + self._player.video_set_spu_delay(delay) def set_volume(self, volume): - def impl(): - self._player.set_volume(self._downscale(volume)) - - self._exec_command(impl) + self._player.audio_set_volume(volume) def seek(self, duration): - def impl(): - self._player.seek(duration) - - self._exec_command(impl) - - def _downscale(self, volume): - return volume / 100 - - def _exec_command(self, command): - with self._player_lock: - if self._player is None: - raise PlayerError("the player is not started") - command() - - def _dispatch(self, event): - self._evt_dispatcher.dispatch(event) + current_time = self._player.get_time() + if current_time != -1: + self._player.set_time(current_time + duration) - def _on_exit(self, player, code): - with self._player_lock: - evt = e.PlayerStopped(self._stop_operation_id) - self._stop_operation_id = None - self._player = None - self._dispatch(evt) + def _on_media_end(self, event): + evt = e.MediaEndReached(None) + self._evt_dispatcher.dispatch(evt) diff --git a/pyproject.toml b/pyproject.toml index 0d1d20c4..78098a34 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -16,17 +16,16 @@ keywords = ['raspberry', 'opencast', 'chromecast'] python = "^3.7" toml = "^0.9" bottle = "^0.12.18" -omxplayer-wrapper = "^0.3.3" youtube-dl = ">= 2020.3.24" pyyaml = "^5.3.1" "hurry.filesize" = "^0.9" -psutil = "^5.7.0" ffmpeg-python = "^0.2.0" transitions = "^0.8.1" structlog = "^20.1.0" colorama = "^0.4.3" sphinx = "^3.0.3" sphinx_rtd_theme = "^0.4.3" +python-vlc = "^3.0.9113" [tool.poetry.dev-dependencies] pytest = "^3.0" diff --git a/setup.sh b/setup.sh index e69e666a..0e52987d 100755 --- a/setup.sh +++ b/setup.sh @@ -44,7 +44,7 @@ function install_system_deps() { info "Installing system dependencies..." sudo apt-get update - sudo apt-get install -y curl lsof python python3 python3-venv python3-pip libdbus-glib-1-dev libdbus-1-dev || + sudo apt-get install -y curl lsof python python3 python3-venv python3-pip || error "failed to install dependencies" curl -sSL "https://raw.githubusercontent.com/python-poetry/poetry/master/get-poetry.py" | python3 } diff --git a/test/integration/app/service/test_player.py b/test/integration/app/service/test_player.py index 0d45f62f..a4f978f2 100644 --- a/test/integration/app/service/test_player.py +++ b/test/integration/app/service/test_player.py @@ -1,4 +1,3 @@ -from test.shared.infra.media.player_mock import make_player_mock from unittest.mock import Mock from OpenCast.app.command import player as Cmd @@ -14,7 +13,7 @@ class PlayerServiceTest(ServiceTestCase): def setUp(self): super(PlayerServiceTest, self).setUp() - self.player = make_player_mock(self.app_facade.evt_dispatcher) + self.player = Mock() media_factory = self.infra_facade.media_factory media_factory.make_player = Mock(return_value=self.player) @@ -82,9 +81,9 @@ def test_next_video(self): ).populate(self.data_facade) next_video_id = IdentityService.id_video("source2") - self.evt_expecter.expect(Evt.PlayerStopped).expect( - Evt.PlayerStarted, next_video_id - ).from_(Cmd.NextVideo, self.player_id) + self.evt_expecter.expect(Evt.PlayerStarted, next_video_id).from_( + Cmd.NextVideo, self.player_id + ) def test_prev_video(self): self.data_producer.player().video("source", None).video( @@ -92,9 +91,9 @@ def test_prev_video(self): ).play().populate(self.data_facade) prev_video_id = IdentityService.id_video("source") - self.evt_expecter.expect(Evt.PlayerStopped).expect( - Evt.PlayerStarted, prev_video_id - ).from_(Cmd.PrevVideo, self.player_id) + self.evt_expecter.expect(Evt.PlayerStarted, prev_video_id).from_( + Cmd.PrevVideo, self.player_id + ) def test_toggle_subtitle(self): self.data_producer.player().populate(self.data_facade) diff --git a/test/shared/infra/media/player_mock.py b/test/shared/infra/media/player_mock.py deleted file mode 100644 index e01c4c30..00000000 --- a/test/shared/infra/media/player_mock.py +++ /dev/null @@ -1,12 +0,0 @@ -from unittest.mock import Mock - -from OpenCast.infra.event.player import PlayerStopped - - -def make_player_mock(evt_dispatcher): - def dispatch_stopped(op_id): - evt_dispatcher.dispatch(PlayerStopped(op_id)) - - player = Mock() - player.stop = Mock(side_effect=dispatch_stopped) - return player From c32cc2b0012e01ded9d7459609f58e0a91cdb2d5 Mon Sep 17 00:00:00 2001 From: Luc Sinet Date: Wed, 3 Jun 2020 19:05:47 +0200 Subject: [PATCH 09/29] replace-omxplayer #comment Move the infra event handling logic from the service to the controller. - The main motivation for this is that vlc distinguishes the player stopping and reaching the end of the media. --- OpenCast/app/controller/controller.py | 10 ++++++++ OpenCast/app/controller/module.py | 2 ++ OpenCast/app/controller/player.py | 33 +++++++++++++++++++++++++++ OpenCast/app/service/player.py | 18 --------------- 4 files changed, 45 insertions(+), 18 deletions(-) create mode 100644 OpenCast/app/controller/player.py diff --git a/OpenCast/app/controller/controller.py b/OpenCast/app/controller/controller.py index 79fd0bc6..ded4d6cb 100644 --- a/OpenCast/app/controller/controller.py +++ b/OpenCast/app/controller/controller.py @@ -1,4 +1,7 @@ +import inspect + from OpenCast.domain.service.identity import IdentityService +from OpenCast.util.naming import name_handler_method class Controller: @@ -11,6 +14,13 @@ def _dispatch(self, cmd_cls, component_id, *args, **kwargs): cmd_id = IdentityService.id_command(cmd_cls, component_id) self._cmd_dispatcher.dispatch(cmd_cls(cmd_id, component_id, *args, **kwargs)) + def _observe(self, module): + classes = inspect.getmembers(module, inspect.isclass) + for _, cls in classes: + if cls.__module__ == module.__name__: + handler_name = name_handler_method(cls) + self._evt_dispatcher.observe(None, {cls: getattr(self, handler_name)}) + def _start_workflow(self, workflow_cls, resource_id, *args, **kwargs): workflow_id = IdentityService.id_workflow(workflow_cls, resource_id) workflow = workflow_cls(workflow_id, self._app_facade, *args, **kwargs) diff --git a/OpenCast/app/controller/module.py b/OpenCast/app/controller/module.py index 90e105fd..d7f1a6d4 100644 --- a/OpenCast/app/controller/module.py +++ b/OpenCast/app/controller/module.py @@ -1,4 +1,5 @@ from .file import FileController +from .player import PlayerController from .player_monitor import PlayerMonitController @@ -8,3 +9,4 @@ def __init__(self, app_facade, infra_facade, data_facade, service_factory): self._player_monitor = PlayerMonitController( app_facade, infra_facade, data_facade, service_factory ) + self._player_controller = PlayerController(app_facade, data_facade) diff --git a/OpenCast/app/controller/player.py b/OpenCast/app/controller/player.py new file mode 100644 index 00000000..3d4c37c0 --- /dev/null +++ b/OpenCast/app/controller/player.py @@ -0,0 +1,33 @@ +import structlog +from OpenCast.app.command import player as Cmd +from OpenCast.domain.service.identity import IdentityService +from OpenCast.infra.event import player as player_events + +from .controller import Controller + + +class PlayerController(Controller): + def __init__(self, app_facade, data_facade): + super(PlayerController, self).__init__(app_facade) + + self._logger = structlog.get_logger(__name__) + self._player_repo = data_facade.player_repo + self._video_repo = data_facade.video_repo + + self._observe(player_events) + + # Infra event handler interface implementation + + def _media_end_reached(self, evt): + model = self._player_repo.get_player() + video = model.next_video() + if video is None: + self._dispatch(Cmd.StopVideo) + else: + self._dispatch(Cmd.PlayVideo, video.id) + + def _dispatch(self, cmd_cls, *args, **kwargs): + player_id = IdentityService.id_player() + return super(PlayerController, self)._dispatch( + cmd_cls, player_id, *args, **kwargs + ) diff --git a/OpenCast/app/service/player.py b/OpenCast/app/service/player.py index 1d90fdc8..f42f62d7 100644 --- a/OpenCast/app/service/player.py +++ b/OpenCast/app/service/player.py @@ -15,24 +15,6 @@ def __init__(self, app_facade, data_facade, media_factory): self._video_repo = data_facade.video_repo self._player = media_factory.make_player(app_facade.evt_dispatcher) - # Infra event handler interface implementation - - def _player_stopped(self, evt): - def stop_player(model): - model.stop() - - def play_next(model): - video = model.next_video() - if video is not None: - self._player.play(video, model.volume) - model.play(video) - - self._update(evt.id, stop_player) - if evt.id is not None: - return - - self._update(evt.id, play_next) - # Command handler interface implementation def _play_video(self, cmd): From 012981635df6a3fc4846ee7a9656d3cf84f8f01b Mon Sep 17 00:00:00 2001 From: Luc Sinet Date: Wed, 3 Jun 2020 20:23:22 +0200 Subject: [PATCH 10/29] replace-omxplayer #comment Set default volume to 70%. - Pass volume value in the event so that it is logged. --- OpenCast/app/service/player.py | 3 +++ OpenCast/domain/event/player.py | 2 +- OpenCast/domain/model/player.py | 4 ++-- test/integration/app/service/test_player.py | 15 ++++++++------- test/unit/domain/model/test_player.py | 2 +- 5 files changed, 15 insertions(+), 11 deletions(-) diff --git a/OpenCast/app/service/player.py b/OpenCast/app/service/player.py index f42f62d7..56f7b194 100644 --- a/OpenCast/app/service/player.py +++ b/OpenCast/app/service/player.py @@ -15,6 +15,9 @@ def __init__(self, app_facade, data_facade, media_factory): self._video_repo = data_facade.video_repo self._player = media_factory.make_player(app_facade.evt_dispatcher) + model = self._player_model() + self._player.set_volume(model.volume) + # Command handler interface implementation def _play_video(self, cmd): diff --git a/OpenCast/domain/event/player.py b/OpenCast/domain/event/player.py index 789f3fe7..f0a2cf6f 100644 --- a/OpenCast/domain/event/player.py +++ b/OpenCast/domain/event/player.py @@ -36,7 +36,7 @@ class VideoSeeked(Event): @dataclass class VolumeUpdated(Event): - pass + volume: int @dataclass diff --git a/OpenCast/domain/model/player.py b/OpenCast/domain/model/player.py index 23cf1922..779f1047 100644 --- a/OpenCast/domain/model/player.py +++ b/OpenCast/domain/model/player.py @@ -20,7 +20,7 @@ def __init__(self, id_): self._index = 0 self._sub_state = False self._sub_delay = 0 - self._volume = 100 + self._volume = 70 def __repr__(self): base_repr = super(Player, self).__repr__() @@ -115,4 +115,4 @@ def subtitle_delay(self, delay): @volume.setter def volume(self, v): self._volume = max(min(200, v), 0) - self._record(Evt.VolumeUpdated) + self._record(Evt.VolumeUpdated, self._volume) diff --git a/test/integration/app/service/test_player.py b/test/integration/app/service/test_player.py index a4f978f2..d9d6df83 100644 --- a/test/integration/app/service/test_player.py +++ b/test/integration/app/service/test_player.py @@ -17,13 +17,13 @@ def setUp(self): media_factory = self.infra_facade.media_factory media_factory.make_player = Mock(return_value=self.player) + self.data_producer.player().populate(self.data_facade) self.service = PlayerService(self.app_facade, self.data_facade, media_factory) self.player_repo = self.data_facade.player_repo self.video_repo = self.data_facade.video_repo self.player_id = IdentityService.id_player() - self.data_producer.player().populate(self.data_facade) def test_play_video(self): self.data_producer.video("source", None).populate(self.data_facade) @@ -71,9 +71,10 @@ def test_change_video_volume(self): self.data_facade ) - self.evt_expecter.expect(Evt.VolumeUpdated).from_( - Cmd.ChangeVolume, self.player_id, Player.VOLUME_STEP - ) + player = self.player_repo.get_player() + self.evt_expecter.expect( + Evt.VolumeUpdated, player.volume + Player.VOLUME_STEP + ).from_(Cmd.ChangeVolume, self.player_id, Player.VOLUME_STEP) def test_next_video(self): self.data_producer.player().video("source", None).play().video( @@ -96,21 +97,21 @@ def test_prev_video(self): ) def test_toggle_subtitle(self): - self.data_producer.player().populate(self.data_facade) + self.data_producer.populate(self.data_facade) self.evt_expecter.expect(Evt.SubtitleStateUpdated).from_( Cmd.ToggleSubtitle, self.player_id ) def test_increase_subtitle_delay(self): - self.data_producer.player().populate(self.data_facade) + self.data_producer.populate(self.data_facade) self.evt_expecter.expect(Evt.SubtitleDelayUpdated).from_( Cmd.IncreaseSubtitleDelay, self.player_id, Player.SUBTITLE_DELAY_STEP ) def test_decrease_subtitle_delay(self): - self.data_producer.player().populate(self.data_facade) + self.data_producer.populate(self.data_facade) self.evt_expecter.expect(Evt.SubtitleDelayUpdated).from_( Cmd.DecreaseSubtitleDelay, self.player_id, Player.SUBTITLE_DELAY_STEP diff --git a/test/unit/domain/model/test_player.py b/test/unit/domain/model/test_player.py index 1987adc2..7d5dfca4 100644 --- a/test/unit/domain/model/test_player.py +++ b/test/unit/domain/model/test_player.py @@ -20,7 +20,7 @@ def make_video(self, source="source_1"): return Video(IdentityService.id_video(source), source, None) def test_construction(self): - self.assertEqual(100, self.player.volume) + self.assertEqual(70, self.player.volume) self.assertFalse(self.player.subtitle_state) self.assertEqual(0, self.player.subtitle_delay) self.assertEqual(PlayerState.STOPPED, self.player.state) From d2bd78bacfae738d43e38d4de6c7a2cb944e6701 Mon Sep 17 00:00:00 2001 From: Luc Sinet Date: Wed, 3 Jun 2020 20:24:46 +0200 Subject: [PATCH 11/29] replace-omxplayer #comment Ensure next and prev don't index the queue out of its bounds. --- OpenCast/domain/model/player.py | 4 +++- test/unit/domain/model/test_player.py | 7 +++++++ 2 files changed, 10 insertions(+), 1 deletion(-) diff --git a/OpenCast/domain/model/player.py b/OpenCast/domain/model/player.py index 779f1047..fd4c6b6a 100644 --- a/OpenCast/domain/model/player.py +++ b/OpenCast/domain/model/player.py @@ -68,13 +68,15 @@ def unpause(self): def next_video(self): if self._index + 1 >= len(self._queue): - if config["player.loop_last"] is True: + if self._queue and config["player.loop_last"] is True: return self._queue[self._index] return None return self._queue[self._index + 1] def prev_video(self): + if not self._queue: + return None if self._index == 0: return self._queue[0] return self._queue[self._index - 1] diff --git a/test/unit/domain/model/test_player.py b/test/unit/domain/model/test_player.py index 7d5dfca4..d6c5785e 100644 --- a/test/unit/domain/model/test_player.py +++ b/test/unit/domain/model/test_player.py @@ -60,6 +60,10 @@ def test_next(self): self.player.play(self.player.next_video()) self.assertEqual(None, self.player.next_video()) + def test_next_no_video(self): + config.load_from_dict({"player": {"loop_last": True}}) + self.assertEqual(None, self.player.next_video()) + def test_prev(self): videos = self.make_videos(video_count=2) for video in videos: @@ -68,6 +72,9 @@ def test_prev(self): self.player.play(self.player.next_video()) self.assertEqual(videos[0], self.player.prev_video()) + def test_prev_no_video(self): + self.assertEqual(None, self.player.prev_video()) + def test_play(self): video = self.make_video() self.player.queue(video) From 5af5421e13d99fa4f22308007fbd1e5011733a01 Mon Sep 17 00:00:00 2001 From: Luc Sinet Date: Wed, 3 Jun 2020 20:25:47 +0200 Subject: [PATCH 12/29] replace-omxplayer #comment Rename StopVideo to StopPlayer. --- OpenCast/app/command/player.py | 2 +- OpenCast/app/controller/player.py | 2 +- OpenCast/app/controller/player_monitor.py | 2 +- OpenCast/app/service/player.py | 2 +- test/integration/app/service/test_player.py | 6 ++++-- 5 files changed, 8 insertions(+), 6 deletions(-) diff --git a/OpenCast/app/command/player.py b/OpenCast/app/command/player.py index 1aab9150..4dc4c641 100644 --- a/OpenCast/app/command/player.py +++ b/OpenCast/app/command/player.py @@ -14,7 +14,7 @@ class QueueVideo(Command): @command -class StopVideo(Command): +class StopPlayer(Command): pass diff --git a/OpenCast/app/controller/player.py b/OpenCast/app/controller/player.py index 3d4c37c0..cdd65293 100644 --- a/OpenCast/app/controller/player.py +++ b/OpenCast/app/controller/player.py @@ -22,7 +22,7 @@ def _media_end_reached(self, evt): model = self._player_repo.get_player() video = model.next_video() if video is None: - self._dispatch(Cmd.StopVideo) + self._dispatch(Cmd.StopPlayer) else: self._dispatch(Cmd.PlayVideo, video.id) diff --git a/OpenCast/app/controller/player_monitor.py b/OpenCast/app/controller/player_monitor.py index 36f8ea8a..c377bfc2 100755 --- a/OpenCast/app/controller/player_monitor.py +++ b/OpenCast/app/controller/player_monitor.py @@ -80,7 +80,7 @@ def _video(self): if control == "pause": self._dispatch(Cmd.ToggleVideoState) elif control == "stop": - self._dispatch(Cmd.StopVideo) + self._dispatch(Cmd.StopPlayer) elif control == "right": self._dispatch(Cmd.SeekVideo, Player.SHORT_TIME_STEP) elif control == "left": diff --git a/OpenCast/app/service/player.py b/OpenCast/app/service/player.py index 56f7b194..83f0d7a7 100644 --- a/OpenCast/app/service/player.py +++ b/OpenCast/app/service/player.py @@ -29,7 +29,7 @@ def _queue_video(self, cmd): video = self._video_repo.get(cmd.video_id) self._queue_video_impl(cmd.id, video) - def _stop_video(self, cmd): + def _stop_player(self, cmd): def stop_video(model): model.stop() self._player.stop() diff --git a/test/integration/app/service/test_player.py b/test/integration/app/service/test_player.py index d9d6df83..d36bcab7 100644 --- a/test/integration/app/service/test_player.py +++ b/test/integration/app/service/test_player.py @@ -41,12 +41,14 @@ def test_queue_video(self): Cmd.QueueVideo, self.player_id, video_id ) - def test_stop_video(self): + def test_stop_player(self): self.data_producer.player().video("source", None).play().populate( self.data_facade ) - self.evt_expecter.expect(Evt.PlayerStopped).from_(Cmd.StopVideo, self.player_id) + self.evt_expecter.expect(Evt.PlayerStopped).from_( + Cmd.StopPlayer, self.player_id + ) def test_pause_video(self): self.data_producer.player().video("source", None).play().populate( From bc43ebfc88218f2d2c98ae3b3e974db8bc28541b Mon Sep 17 00:00:00 2001 From: Luc Sinet Date: Wed, 3 Jun 2020 20:26:14 +0200 Subject: [PATCH 13/29] replace-omxplayer #comment Add player controller tests. --- test/unit/app/controller/__init__.py | 0 test/unit/app/controller/test_player.py | 29 +++++++++++++++++++++++ test/unit/app/controller/util.py | 31 +++++++++++++++++++++++++ 3 files changed, 60 insertions(+) create mode 100644 test/unit/app/controller/__init__.py create mode 100644 test/unit/app/controller/test_player.py create mode 100644 test/unit/app/controller/util.py diff --git a/test/unit/app/controller/__init__.py b/test/unit/app/controller/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/test/unit/app/controller/test_player.py b/test/unit/app/controller/test_player.py new file mode 100644 index 00000000..45c7aac4 --- /dev/null +++ b/test/unit/app/controller/test_player.py @@ -0,0 +1,29 @@ +from OpenCast.app.command import player as Cmd +from OpenCast.app.controller.player import PlayerController +from OpenCast.domain.service.identity import IdentityService +from OpenCast.infra.event import player as Evt + +from .util import ControllerTestCase + + +class PlayerControllerTest(ControllerTestCase): + def setUp(self): + super(PlayerControllerTest, self).setUp() + + self.data_producer.player().populate(self.data_facade) + self.controller = PlayerController(self.app_facade, self.data_facade) + + def test_media_end_reached_without_video(self): + self.raise_event(self.controller, Evt.MediaEndReached, None) + self.expect_dispatch(Cmd.StopPlayer, IdentityService.id_player()) + + def test_media_end_reached_with_videos(self): + self.data_producer.player().video("source", None).video( + "next_video", None + ).populate(self.data_facade) + self.raise_event(self.controller, Evt.MediaEndReached, None) + self.expect_dispatch( + Cmd.PlayVideo, + IdentityService.id_player(), + IdentityService.id_video("next_video"), + ) diff --git a/test/unit/app/controller/util.py b/test/unit/app/controller/util.py new file mode 100644 index 00000000..5bb0bde5 --- /dev/null +++ b/test/unit/app/controller/util.py @@ -0,0 +1,31 @@ +import uuid +from test.shared.app.facade_mock import AppFacadeMock +from test.shared.infra.data.producer import DataProducer +from test.util import TestCase +from unittest.mock import Mock + +from OpenCast.app.service.error import OperationError +from OpenCast.domain.service.identity import IdentityService +from OpenCast.infra.data.facade import DataFacade +from OpenCast.infra.data.repo.factory import RepoFactory +from OpenCast.util.naming import name_factory_method, name_handler_method + + +class ControllerTestCase(TestCase): + def setUp(self): + self.app_facade = AppFacadeMock() + + repo_factory = RepoFactory() + self.data_facade = DataFacade(repo_factory) + + self.data_producer = DataProducer.make() + + def expect_dispatch(self, cmd_cls, model_id, *args, **kwargs): + cmd_id = IdentityService.id_command(cmd_cls, model_id) + cmd = cmd_cls(cmd_id, model_id, *args, **kwargs) + self.app_facade.cmd_dispatcher.dispatch.assert_called_once_with(cmd) + return cmd + + def raise_event(self, controller, evt_cls, *args, **kwargs): + event = evt_cls(*args, **kwargs) + getattr(controller, name_handler_method(evt_cls))(event) From 7eaf472a934d0fbbfddee7db825ce7a7cb8129c0 Mon Sep 17 00:00:00 2001 From: Luc Sinet Date: Wed, 3 Jun 2020 20:34:03 +0200 Subject: [PATCH 14/29] replace-omxplayer #comment Remove subtitle manipulation logic as not required by vlc. --- OpenCast/domain/service/factory.py | 4 +-- OpenCast/domain/service/subtitle.py | 16 ++--------- OpenCast/infra/service/factory.py | 6 +--- OpenCast/infra/service/subtitle.py | 34 ----------------------- test/unit/domain/service/test_subtitle.py | 2 +- 5 files changed, 6 insertions(+), 56 deletions(-) delete mode 100644 OpenCast/infra/service/subtitle.py diff --git a/OpenCast/domain/service/factory.py b/OpenCast/domain/service/factory.py index a04c2226..2e096960 100644 --- a/OpenCast/domain/service/factory.py +++ b/OpenCast/domain/service/factory.py @@ -10,6 +10,4 @@ def make_source_service(self, *args): return SourceService(*args) def make_subtitle_service(self, *args): - return SubtitleService( - self._infra_service_factory.make_subtitle_converter(), *args - ) + return SubtitleService(*args) diff --git a/OpenCast/domain/service/subtitle.py b/OpenCast/domain/service/subtitle.py index c27376a1..376354b5 100644 --- a/OpenCast/domain/service/subtitle.py +++ b/OpenCast/domain/service/subtitle.py @@ -4,9 +4,8 @@ class SubtitleService: - def __init__(self, subtitle_converter, ffmpeg_wrapper, downloader): + def __init__(self, ffmpeg_wrapper, downloader): self._logger = structlog.get_logger(__name__) - self._subtitle_converter = subtitle_converter self._ffmpeg_wrapper = ffmpeg_wrapper self._downloader = downloader @@ -52,13 +51,7 @@ def _load_from_disk(self, video_path: Path, language: str) -> str: and stream["tags"]["language"] == language ): self._logger.debug(f"Match: {subtitle}") - if self._ffmpeg_wrapper.extract_stream( - src=video_path, - dest=subtitle, - stream_idx=stream["index"], - override=False, - ): - return subtitle + return subtitle return None def _download_from_source( @@ -68,7 +61,4 @@ def _download_from_source( subtitle = self._downloader.download_subtitle( video_source, dest, language, ["vtt"] ) - if subtitle is None or Path(subtitle).suffix == "srt": - return subtitle - - return self._subtitle_converter.vtt_to_srt(Path(subtitle)) + return subtitle diff --git a/OpenCast/infra/service/factory.py b/OpenCast/infra/service/factory.py index c6e08fe6..f3d0c996 100644 --- a/OpenCast/infra/service/factory.py +++ b/OpenCast/infra/service/factory.py @@ -1,6 +1,2 @@ -from .subtitle import SubtitleConverter - - class ServiceFactory: - def make_subtitle_converter(self, *args): - return SubtitleConverter(*args) + pass diff --git a/OpenCast/infra/service/subtitle.py b/OpenCast/infra/service/subtitle.py deleted file mode 100644 index b8177d1e..00000000 --- a/OpenCast/infra/service/subtitle.py +++ /dev/null @@ -1,34 +0,0 @@ -from pathlib import Path - - -class SubtitleConverter: - def vtt_to_srt(self, subtitle: Path): - print(f"subtitle: {subtitle}") - dest = subtitle.with_suffix(".srt") - with open(subtitle, "r") as fp: - content = fp.readlines() - - output = [] - subtitle_count = 0 - in_caption = False - for line in content: - if "-->" in line: - tokens = line.split(" --> ") - if len(tokens) < 3: - continue - - in_caption = True - if subtitle_count > 0: - output.append("\n") - subtitle_count += 1 - output.append(f"{subtitle_count}\n") - output.append(line.replace(".", ",")) - elif line != "" and in_caption: # Skip vtt header and notes - output.append(line) - else: - in_caption = False - - with open(dest, "w") as fp: - fp.writelines(output) - - return dest diff --git a/test/unit/domain/service/test_subtitle.py b/test/unit/domain/service/test_subtitle.py index b13d2489..35334469 100644 --- a/test/unit/domain/service/test_subtitle.py +++ b/test/unit/domain/service/test_subtitle.py @@ -5,7 +5,7 @@ class SubtitleServiceTest(TestCase): def setUp(self): - self._service = SubtitleService(None, None, None) + self._service = SubtitleService(None, None) def test_load_from_disk(self): pass From 4881c53407b90f628c3a2cfb7bee543bd52090c3 Mon Sep 17 00:00:00 2001 From: Luc Sinet Date: Wed, 3 Jun 2020 20:35:12 +0200 Subject: [PATCH 15/29] replace-omxplayer #comment Update dependencies. - sphinx (3.0.3 -> 3.0.4) - youtube-dl (2020.5.8 -> 2020.5.29) --- poetry.lock | 122 ++++++++-------------------------------------------- 1 file changed, 19 insertions(+), 103 deletions(-) diff --git a/poetry.lock b/poetry.lock index 02ecf27a..9b663055 100644 --- a/poetry.lock +++ b/poetry.lock @@ -82,22 +82,6 @@ version = "5.1" [package.extras] toml = ["toml"] -[[package]] -category = "main" -description = "Python bindings for libdbus" -name = "dbus-python" -optional = false -python-versions = "*" -version = "1.2.16" - -[[package]] -category = "main" -description = "Decorators for Humans" -name = "decorator" -optional = false -python-versions = ">=2.6, !=3.0.*, !=3.1.*" -version = "4.4.2" - [[package]] category = "main" description = "Docutils -- Python Documentation Utilities" @@ -106,14 +90,6 @@ optional = false python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*, !=3.4.*" version = "0.16" -[[package]] -category = "main" -description = "Observer pattern made muy facil" -name = "evento" -optional = false -python-versions = "*" -version = "1.0.2" - [[package]] category = "main" description = "Python bindings for FFmpeg - with complex filtering support" @@ -209,24 +185,6 @@ optional = false python-versions = ">=3.5" version = "8.3.0" -[[package]] -category = "main" -description = "A library for controlling omxplayer on the Raspberry Pi" -name = "omxplayer-wrapper" -optional = false -python-versions = "*" -version = "0.3.3" - -[package.dependencies] -dbus-python = "*" -decorator = "*" -evento = "*" -pathlib2 = "*" - -[package.extras] -docs = ["sphinx", "alabaster", "pygments"] -test = ["mock", "pytest", "pytest-cov", "nose", "parameterized"] - [[package]] category = "main" description = "Core utilities for Python packages" @@ -239,17 +197,6 @@ version = "20.4" pyparsing = ">=2.0.2" six = "*" -[[package]] -category = "main" -description = "Object-oriented filesystem paths" -name = "pathlib2" -optional = false -python-versions = "*" -version = "2.3.5" - -[package.dependencies] -six = "*" - [[package]] category = "dev" description = "plugin and hook calling mechanisms for python" @@ -266,17 +213,6 @@ version = ">=0.12" [package.extras] dev = ["pre-commit", "tox"] -[[package]] -category = "main" -description = "Cross-platform lib for process and system monitoring in Python." -name = "psutil" -optional = false -python-versions = ">=2.6, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*" -version = "5.7.0" - -[package.extras] -enum = ["enum34"] - [[package]] category = "dev" description = "library with cross-python path, ini-parsing, io, code, log facilities" @@ -334,6 +270,14 @@ pytest = ">=3.6" [package.extras] testing = ["fields", "hunter", "process-tests (2.0.2)", "six", "pytest-xdist", "virtualenv"] +[[package]] +category = "main" +description = "VLC bindings for python." +name = "python-vlc" +optional = false +python-versions = "*" +version = "3.0.9113" + [[package]] category = "main" description = "World timezone definitions, modern and historical" @@ -390,7 +334,7 @@ description = "Python documentation generator" name = "sphinx" optional = false python-versions = ">=3.5" -version = "3.0.3" +version = "3.0.4" [package.dependencies] Jinja2 = ">=2.3" @@ -557,7 +501,7 @@ description = "YouTube video downloader" name = "youtube-dl" optional = false python-versions = "*" -version = "2020.5.8" +version = "2020.5.29" [[package]] category = "dev" @@ -573,7 +517,7 @@ docs = ["sphinx", "jaraco.packaging (>=3.2)", "rst.linker (>=1.9)"] testing = ["jaraco.itertools", "func-timeout"] [metadata] -content-hash = "9b678dad850bc124a93387d7640940365d3f7d35cddd2d637ec9c793aed3af8a" +content-hash = "9241af1b3c4d60c2a60e5cf809f9309cb00b88f435fd025fadd6bdbc43639027" python-versions = "^3.7" [metadata.files] @@ -642,21 +586,10 @@ coverage = [ {file = "coverage-5.1-cp39-cp39-win_amd64.whl", hash = "sha256:bb28a7245de68bf29f6fb199545d072d1036a1917dca17a1e75bbb919e14ee8e"}, {file = "coverage-5.1.tar.gz", hash = "sha256:f90bfc4ad18450c80b024036eaf91e4a246ae287701aaa88eaebebf150868052"}, ] -dbus-python = [ - {file = "dbus-python-1.2.16.tar.gz", hash = "sha256:11238f1d86c995d8aed2e22f04a1e3779f0d70e587caffeab4857f3c662ed5a4"}, -] -decorator = [ - {file = "decorator-4.4.2-py2.py3-none-any.whl", hash = "sha256:41fa54c2a0cc4ba648be4fd43cff00aedf5b9465c9bf18d64325bc225f08f760"}, - {file = "decorator-4.4.2.tar.gz", hash = "sha256:e3a62f0520172440ca0dcc823749319382e377f37f140a0b99ef45fecb84bfe7"}, -] docutils = [ {file = "docutils-0.16-py2.py3-none-any.whl", hash = "sha256:0c5b78adfbf7762415433f5515cd5c9e762339e23369dbe8000d84a4bf4ab3af"}, {file = "docutils-0.16.tar.gz", hash = "sha256:c2de3a60e9e7d07be26b7f2b00ca0309c207e06c100f9cc2a94931fc75a478fc"}, ] -evento = [ - {file = "evento-1.0.2-py3-none-any.whl", hash = "sha256:7a010c2a7d64b84dff06fb56a28d7e64b9234b15778bf96d526d6d839f181547"}, - {file = "evento-1.0.2.tar.gz", hash = "sha256:093c054ae590b968c3c2c7852d02ddbaea25fc1305024f83dd636fcba3545871"}, -] ffmpeg-python = [ {file = "ffmpeg-python-0.2.0.tar.gz", hash = "sha256:65225db34627c578ef0e11c8b1eb528bb35e024752f6f10b78c011f6f64c4127"}, {file = "ffmpeg_python-0.2.0-py3-none-any.whl", hash = "sha256:ac441a0404e053f8b6a1113a77c0f452f1cfc62f6344a769475ffdc0f56c23c5"}, @@ -722,35 +655,14 @@ more-itertools = [ {file = "more-itertools-8.3.0.tar.gz", hash = "sha256:558bb897a2232f5e4f8e2399089e35aecb746e1f9191b6584a151647e89267be"}, {file = "more_itertools-8.3.0-py3-none-any.whl", hash = "sha256:7818f596b1e87be009031c7653d01acc46ed422e6656b394b0f765ce66ed4982"}, ] -omxplayer-wrapper = [ - {file = "omxplayer-wrapper-0.3.3.tar.gz", hash = "sha256:05b64036a90220707608c5dd958428dd3805d544c1be5d2d26f6bec10a805b6d"}, - {file = "omxplayer_wrapper-0.3.3-py2.py3-none-any.whl", hash = "sha256:0b56408c91d075298b88209fb68ef96ab8ac25bafd8b4d47c314987e460a1dad"}, -] packaging = [ {file = "packaging-20.4-py2.py3-none-any.whl", hash = "sha256:998416ba6962ae7fbd6596850b80e17859a5753ba17c32284f67bfff33784181"}, {file = "packaging-20.4.tar.gz", hash = "sha256:4357f74f47b9c12db93624a82154e9b120fa8293699949152b22065d556079f8"}, ] -pathlib2 = [ - {file = "pathlib2-2.3.5-py2.py3-none-any.whl", hash = "sha256:0ec8205a157c80d7acc301c0b18fbd5d44fe655968f5d947b6ecef5290fc35db"}, - {file = "pathlib2-2.3.5.tar.gz", hash = "sha256:6cd9a47b597b37cc57de1c05e56fb1a1c9cc9fab04fe78c29acd090418529868"}, -] pluggy = [ {file = "pluggy-0.13.1-py2.py3-none-any.whl", hash = "sha256:966c145cd83c96502c3c3868f50408687b38434af77734af1e9ca461a4081d2d"}, {file = "pluggy-0.13.1.tar.gz", hash = "sha256:15b2acde666561e1298d71b523007ed7364de07029219b604cf808bfa1c765b0"}, ] -psutil = [ - {file = "psutil-5.7.0-cp27-none-win32.whl", hash = "sha256:298af2f14b635c3c7118fd9183843f4e73e681bb6f01e12284d4d70d48a60953"}, - {file = "psutil-5.7.0-cp27-none-win_amd64.whl", hash = "sha256:75e22717d4dbc7ca529ec5063000b2b294fc9a367f9c9ede1f65846c7955fd38"}, - {file = "psutil-5.7.0-cp35-cp35m-win32.whl", hash = "sha256:f344ca230dd8e8d5eee16827596f1c22ec0876127c28e800d7ae20ed44c4b310"}, - {file = "psutil-5.7.0-cp35-cp35m-win_amd64.whl", hash = "sha256:e2d0c5b07c6fe5a87fa27b7855017edb0d52ee73b71e6ee368fae268605cc3f5"}, - {file = "psutil-5.7.0-cp36-cp36m-win32.whl", hash = "sha256:a02f4ac50d4a23253b68233b07e7cdb567bd025b982d5cf0ee78296990c22d9e"}, - {file = "psutil-5.7.0-cp36-cp36m-win_amd64.whl", hash = "sha256:1413f4158eb50e110777c4f15d7c759521703bd6beb58926f1d562da40180058"}, - {file = "psutil-5.7.0-cp37-cp37m-win32.whl", hash = "sha256:d008ddc00c6906ec80040d26dc2d3e3962109e40ad07fd8a12d0284ce5e0e4f8"}, - {file = "psutil-5.7.0-cp37-cp37m-win_amd64.whl", hash = "sha256:73f35ab66c6c7a9ce82ba44b1e9b1050be2a80cd4dcc3352cc108656b115c74f"}, - {file = "psutil-5.7.0-cp38-cp38-win32.whl", hash = "sha256:60b86f327c198561f101a92be1995f9ae0399736b6eced8f24af41ec64fb88d4"}, - {file = "psutil-5.7.0-cp38-cp38-win_amd64.whl", hash = "sha256:d84029b190c8a66a946e28b4d3934d2ca1528ec94764b180f7d6ea57b0e75e26"}, - {file = "psutil-5.7.0.tar.gz", hash = "sha256:685ec16ca14d079455892f25bd124df26ff9137664af445563c1bd36629b5e0e"}, -] py = [ {file = "py-1.8.1-py2.py3-none-any.whl", hash = "sha256:c20fdd83a5dbc0af9efd622bee9a5564e278f6380fffcacc43ba6f43db2813b0"}, {file = "py-1.8.1.tar.gz", hash = "sha256:5e27081401262157467ad6e7f851b7aa402c5852dbcb3dae06768434de5752aa"}, @@ -771,6 +683,10 @@ pytest-cov = [ {file = "pytest-cov-2.9.0.tar.gz", hash = "sha256:b6a814b8ed6247bd81ff47f038511b57fe1ce7f4cc25b9106f1a4b106f1d9322"}, {file = "pytest_cov-2.9.0-py2.py3-none-any.whl", hash = "sha256:c87dfd8465d865655a8213859f1b4749b43448b5fae465cb981e16d52a811424"}, ] +python-vlc = [ + {file = "python-vlc-3.0.9113.tar.gz", hash = "sha256:5422b79d347b6419008ee91cfd9663edc37eaf2a0bd8fb9017d4cc2e5f249dda"}, + {file = "python_vlc-3.0.9113-py3-none-any.whl", hash = "sha256:6e6799cae79a1cda737144f6ac72d72d449752c5384fb2b5cb326600626e3bef"}, +] pytz = [ {file = "pytz-2020.1-py2.py3-none-any.whl", hash = "sha256:a494d53b6d39c3c6e44c3bec237336e14305e4f29bbf800b599253057fbb79ed"}, {file = "pytz-2020.1.tar.gz", hash = "sha256:c35965d010ce31b23eeb663ed3cc8c906275d6be1a34393a1d73a41febf4a048"}, @@ -801,8 +717,8 @@ snowballstemmer = [ {file = "snowballstemmer-2.0.0.tar.gz", hash = "sha256:df3bac3df4c2c01363f3dd2cfa78cce2840a79b9f1c2d2de9ce8d31683992f52"}, ] sphinx = [ - {file = "Sphinx-3.0.3-py3-none-any.whl", hash = "sha256:f5505d74cf9592f3b997380f9bdb2d2d0320ed74dd69691e3ee0644b956b8d83"}, - {file = "Sphinx-3.0.3.tar.gz", hash = "sha256:62edfd92d955b868d6c124c0942eba966d54b5f3dcb4ded39e65f74abac3f572"}, + {file = "Sphinx-3.0.4-py3-none-any.whl", hash = "sha256:779a519adbd3a70fc7c468af08c5e74829868b0a5b34587b33340e010291856c"}, + {file = "Sphinx-3.0.4.tar.gz", hash = "sha256:ea64df287958ee5aac46be7ac2b7277305b0381d213728c3a49d8bb9b8415807"}, ] sphinx-rtd-theme = [ {file = "sphinx_rtd_theme-0.4.3-py2.py3-none-any.whl", hash = "sha256:00cf895504a7895ee433807c62094cf1e95f065843bf3acd17037c3e9a2becd4"}, @@ -849,8 +765,8 @@ urllib3 = [ {file = "urllib3-1.25.9.tar.gz", hash = "sha256:3018294ebefce6572a474f0604c2021e33b3fd8006ecd11d62107a5d2a963527"}, ] youtube-dl = [ - {file = "youtube_dl-2020.5.8-py2.py3-none-any.whl", hash = "sha256:0b5d3280522469968eb62eecb1f966f422b2be22f000a801bf87cb2172d8ea39"}, - {file = "youtube_dl-2020.5.8.tar.gz", hash = "sha256:22da6788b55b7b267c6d59bcdfaf10e67a9ac980976d50d29a670473ad2a05bb"}, + {file = "youtube_dl-2020.5.29-py2.py3-none-any.whl", hash = "sha256:2d45840772ecc57e151b0be78dd89e9772b6aa29295746be38abb9c30dad5bb3"}, + {file = "youtube_dl-2020.5.29.tar.gz", hash = "sha256:1a3d84afa851dce2fccc2dfc0f9ffa0e22314ffba6d528b34b4a7fe3e0cf2264"}, ] zipp = [ {file = "zipp-3.1.0-py3-none-any.whl", hash = "sha256:aa36550ff0c0b7ef7fa639055d797116ee891440eac1a56f378e2d3179e0320b"}, From 85b7fc21f760e9f44266ab6c811f8468127e66f4 Mon Sep 17 00:00:00 2001 From: Luc Sinet Date: Fri, 5 Jun 2020 14:31:34 +0200 Subject: [PATCH 16/29] replace-omxplayer #comment Adjust subtitle delay updates in us. --- OpenCast/infra/media/player_wrapper.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/OpenCast/infra/media/player_wrapper.py b/OpenCast/infra/media/player_wrapper.py index 59861931..733631b9 100644 --- a/OpenCast/infra/media/player_wrapper.py +++ b/OpenCast/infra/media/player_wrapper.py @@ -49,7 +49,7 @@ def toggle_subtitle(self): self._player.toggle_teletext() def set_subtitle_delay(self, delay: int): - self._player.video_set_spu_delay(delay) + self._player.video_set_spu_delay(delay * 1000) def set_volume(self, volume): self._player.audio_set_volume(volume) From e11619ebfd565d2c1cacc28dc2dc858569d52056 Mon Sep 17 00:00:00 2001 From: Luc Sinet Date: Fri, 5 Jun 2020 14:38:07 +0200 Subject: [PATCH 17/29] replace-omxplayer #comment Rename video is_file() to from_disk() --- OpenCast/app/service/video.py | 2 +- OpenCast/domain/model/video.py | 6 +++--- OpenCast/domain/service/source.py | 2 +- OpenCast/domain/service/subtitle.py | 2 +- 4 files changed, 6 insertions(+), 6 deletions(-) diff --git a/OpenCast/app/service/video.py b/OpenCast/app/service/video.py index e7995047..33770ee3 100644 --- a/OpenCast/app/service/video.py +++ b/OpenCast/app/service/video.py @@ -54,7 +54,7 @@ def impl(ctx, video): ctx.update(video) video = self._video_repo.get(cmd.model_id) - if video.is_file(): + if video.from_disk(): self._start_transaction(self._video_repo, cmd.id, impl, video) return diff --git a/OpenCast/domain/model/video.py b/OpenCast/domain/model/video.py index 8857cd41..6ee0c80a 100644 --- a/OpenCast/domain/model/video.py +++ b/OpenCast/domain/model/video.py @@ -59,8 +59,8 @@ def subtitle(self, subtitle: str): self._subtitle = subtitle self._record(Evt.VideoSubtitleFetched, self._subtitle) + def from_disk(self): + return Path(self._source).is_file() + def delete(self): self._record(Evt.VideoDeleted) - - def is_file(self): - return Path(self._source).is_file() diff --git a/OpenCast/domain/service/source.py b/OpenCast/domain/service/source.py index 3cb88365..933190b4 100644 --- a/OpenCast/domain/service/source.py +++ b/OpenCast/domain/service/source.py @@ -12,6 +12,6 @@ def unfold(self, source): return self._downloader.unfold_playlist(source) def pick_stream_metadata(self, video): - if video.is_file(): + if video.from_disk(): return {"title": str(Path(video.source).name)} return self._downloader.pick_stream_metadata(video.source, ["title"]) diff --git a/OpenCast/domain/service/subtitle.py b/OpenCast/domain/service/subtitle.py index 376354b5..74a43829 100644 --- a/OpenCast/domain/service/subtitle.py +++ b/OpenCast/domain/service/subtitle.py @@ -16,7 +16,7 @@ def fetch_subtitle( if subtitle is not None: return subtitle - if search_source: + if not video.from_disk(): subtitle = self._download_from_source(video.source, video.path, language) if subtitle is not None: return subtitle From e1fbdbb383b3796847f8b1369591ae778a6dda2b Mon Sep 17 00:00:00 2001 From: Luc Sinet Date: Fri, 5 Jun 2020 14:41:32 +0200 Subject: [PATCH 18/29] replace-omxplayer #comment Inject and share a single VLC instance. --- OpenCast/__init__.py | 8 +++++--- OpenCast/infra/media/factory.py | 6 +++++- OpenCast/infra/media/player_wrapper.py | 14 ++++---------- 3 files changed, 14 insertions(+), 14 deletions(-) diff --git a/OpenCast/__init__.py b/OpenCast/__init__.py index febb60ae..f9423a0e 100644 --- a/OpenCast/__init__.py +++ b/OpenCast/__init__.py @@ -3,6 +3,7 @@ from concurrent.futures import ThreadPoolExecutor import structlog +from vlc import Instance as VlcInstance from .app.controller.module import ControllerModule from .app.facade import AppFacade @@ -42,9 +43,10 @@ def main(argv=None): repo_factory = RepoFactory() data_facade = DataFacade(repo_factory) - downloader_executor = ThreadPoolExecutor(config["downloader.max_concurrency"]) - io_factory = IoFactory(downloader_executor) - media_factory = MediaFactory() + io_factory = IoFactory() + media_factory = MediaFactory( + VlcInstance(), ThreadPoolExecutor(config["downloader.max_concurrency"]) + ) infra_facade = InfraFacade(io_factory, media_factory) ControllerModule(app_facade, infra_facade, data_facade, service_factory) diff --git a/OpenCast/infra/media/factory.py b/OpenCast/infra/media/factory.py index 8d019a1e..73894d68 100644 --- a/OpenCast/infra/media/factory.py +++ b/OpenCast/infra/media/factory.py @@ -4,5 +4,9 @@ class MediaFactory: + def __init__(self, vlc_instance, downloader_executor): + self._downloader_executor = downloader_executor + self._vlc = vlc_instance + def make_player(self, *args): - return PlayerWrapper(*args) + return PlayerWrapper(self._vlc, *args) diff --git a/OpenCast/infra/media/player_wrapper.py b/OpenCast/infra/media/player_wrapper.py index 733631b9..28377325 100644 --- a/OpenCast/infra/media/player_wrapper.py +++ b/OpenCast/infra/media/player_wrapper.py @@ -7,32 +7,26 @@ class PlayerWrapper: - def __init__(self, evt_dispatcher): + def __init__(self, vlc_instance, evt_dispatcher): self._logger = structlog.get_logger(__name__) + + self._instance = vlc_instance self._evt_dispatcher = evt_dispatcher - self._instance = vlc.Instance() self._player = self._instance.media_player_new() - # self._list_player = self._instance.media_list_player_new() - # self._list_player.set_media_player(self._player) - # self._playlist = self._instance.media_list_new() - # self._list_player.set_media_list(self._playlist) self._id_to_media = {} self._lock = Lock() self._stop_operation_id = None player_events = self._player.event_manager() - player_events.event_attach( - vlc.EventType.MediaPlayerEndReached, self._on_media_end - ) + player_events.event_attach(EventType.MediaPlayerEndReached, self._on_media_end) def play(self, video_id: UUID, video_path: str): media = self._id_to_media.get(video_id, None) if media is None: media = self._instance.media_new(video_path) self._id_to_media[video_id] = media - print(f"Play video {video_id}") self._player.set_media(media) self._player.play() From eb0ceeb5152691704ac521fc98ed28a3b69305c6 Mon Sep 17 00:00:00 2001 From: Luc Sinet Date: Fri, 5 Jun 2020 14:49:05 +0200 Subject: [PATCH 19/29] replace-omxplayer #comment Replace the ffmpeg wrapper by a media parser using vlc capabilities. Extracting streams is not mandatory anymore as vlc can play softcoded subtitles. - Use vlc for parsing video data and extracting its streams. - Add tests for the parser. --- OpenCast/infra/io/ffmpeg_wrapper.py | 30 --------------- OpenCast/infra/media/error.py | 4 ++ OpenCast/infra/media/parser.py | 55 ++++++++++++++++++++++++++++ poetry.lock | 31 +--------------- pyproject.toml | 1 - test/unit/infra/media/__init__.py | 0 test/unit/infra/media/test_parser.py | 54 +++++++++++++++++++++++++++ 7 files changed, 114 insertions(+), 61 deletions(-) delete mode 100644 OpenCast/infra/io/ffmpeg_wrapper.py create mode 100644 OpenCast/infra/media/parser.py create mode 100644 test/unit/infra/media/__init__.py create mode 100644 test/unit/infra/media/test_parser.py diff --git a/OpenCast/infra/io/ffmpeg_wrapper.py b/OpenCast/infra/io/ffmpeg_wrapper.py deleted file mode 100644 index 1152f3e9..00000000 --- a/OpenCast/infra/io/ffmpeg_wrapper.py +++ /dev/null @@ -1,30 +0,0 @@ -import ffmpeg -import structlog - - -class FFmpegWrapper: - def __init__(self): - self._logger = structlog.get_logger(__name__) - - def probe(self, file_path): - try: - return ffmpeg.probe(str(file_path)) - except ffmpeg.Error as e: - self._logger.error("Probing error", error=e) - return None - - def extract_stream(self, src, dest, stream_idx, override): - channel = "0:{}".format(stream_idx) - args = [] - if override is True: - args.append("-n") - try: - ffmpeg.input(str(src)).output(str(dest), map=channel).global_args( - *args - ).run() - return dest - except ffmpeg.Error as e: - if e is None and override is True: # The file probably exists - return True - self._logger.error("Extraction error", error=e.stderr) - return False diff --git a/OpenCast/infra/media/error.py b/OpenCast/infra/media/error.py index 5de8951a..0d2579b3 100644 --- a/OpenCast/infra/media/error.py +++ b/OpenCast/infra/media/error.py @@ -1,2 +1,6 @@ class PlayerError(Exception): pass + + +class VideoParsingError(Exception): + pass diff --git a/OpenCast/infra/media/parser.py b/OpenCast/infra/media/parser.py new file mode 100644 index 00000000..5fcc91ef --- /dev/null +++ b/OpenCast/infra/media/parser.py @@ -0,0 +1,55 @@ +from threading import Condition + +import structlog +from vlc import EventType, MediaParsedStatus, MediaParseFlag, TrackType + +from .error import VideoParsingError + + +class VideoParser: + def __init__(self, vlc_instance): + self._logger = structlog.get_logger(__name__) + self._vlc = vlc_instance + + def parse_streams(self, video_path: str): + media = self._vlc.media_new(video_path) + cv = Condition() + + def parse_status_update(_): + with cv: + cv.notify() + + def raise_on_error(): + status = media.get_parsed_status() + if status != MediaParsedStatus.done: + self._logger.error( + "Stream parsing error", video=video_path, status=status + ) + raise VideoParsingError( + f"Can't parse streams from '{video_path}', status='{str(status)}'" + ) + + media.event_manager().event_attach( + EventType.MediaParsedChanged, parse_status_update + ) + with cv: + if media.parse_with_options(MediaParseFlag.local, timeout=5000) == -1: + raise_on_error() + cv.wait_for(media.is_parsed) + + raise_on_error() + streams = media.tracks_get() + + type_to_code = { + TrackType.audio: "audio", + TrackType.video: "video", + TrackType.ext: "subtitle", + } + return [ + ( + stream.id, + type_to_code.get(stream.type, "unknown"), + stream.language.decode("UTF-8"), + ) + for stream in streams + ] diff --git a/poetry.lock b/poetry.lock index 9b663055..e9f7a9db 100644 --- a/poetry.lock +++ b/poetry.lock @@ -90,28 +90,6 @@ optional = false python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*, !=3.4.*" version = "0.16" -[[package]] -category = "main" -description = "Python bindings for FFmpeg - with complex filtering support" -name = "ffmpeg-python" -optional = false -python-versions = "*" -version = "0.2.0" - -[package.dependencies] -future = "*" - -[package.extras] -dev = ["future (0.17.1)", "numpy (1.16.4)", "pytest-mock (1.10.4)", "pytest (4.6.1)", "Sphinx (2.1.0)", "tox (3.12.1)"] - -[[package]] -category = "main" -description = "Clean single-source support for Python 3 and 2" -name = "future" -optional = false -python-versions = ">=2.6, !=3.0.*, !=3.1.*, !=3.2.*" -version = "0.18.2" - [[package]] category = "main" description = "A simple Python library for human readable file sizes (or anything sized in bytes)." @@ -517,7 +495,7 @@ docs = ["sphinx", "jaraco.packaging (>=3.2)", "rst.linker (>=1.9)"] testing = ["jaraco.itertools", "func-timeout"] [metadata] -content-hash = "9241af1b3c4d60c2a60e5cf809f9309cb00b88f435fd025fadd6bdbc43639027" +content-hash = "f2286064f6040a5efa9b8c07f86f9a9ffdfed0f0fcc8fa655109c7b60c869f61" python-versions = "^3.7" [metadata.files] @@ -590,13 +568,6 @@ docutils = [ {file = "docutils-0.16-py2.py3-none-any.whl", hash = "sha256:0c5b78adfbf7762415433f5515cd5c9e762339e23369dbe8000d84a4bf4ab3af"}, {file = "docutils-0.16.tar.gz", hash = "sha256:c2de3a60e9e7d07be26b7f2b00ca0309c207e06c100f9cc2a94931fc75a478fc"}, ] -ffmpeg-python = [ - {file = "ffmpeg-python-0.2.0.tar.gz", hash = "sha256:65225db34627c578ef0e11c8b1eb528bb35e024752f6f10b78c011f6f64c4127"}, - {file = "ffmpeg_python-0.2.0-py3-none-any.whl", hash = "sha256:ac441a0404e053f8b6a1113a77c0f452f1cfc62f6344a769475ffdc0f56c23c5"}, -] -future = [ - {file = "future-0.18.2.tar.gz", hash = "sha256:b1bead90b70cf6ec3f0710ae53a525360fa360d306a86583adc6bf83a4db537d"}, -] "hurry.filesize" = [ {file = "hurry.filesize-0.9.tar.gz", hash = "sha256:f5368329adbef86accd3bc9490522340bb79260455ae89b1a42c10f63801b9a6"}, ] diff --git a/pyproject.toml b/pyproject.toml index 78098a34..6103b31e 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -19,7 +19,6 @@ bottle = "^0.12.18" youtube-dl = ">= 2020.3.24" pyyaml = "^5.3.1" "hurry.filesize" = "^0.9" -ffmpeg-python = "^0.2.0" transitions = "^0.8.1" structlog = "^20.1.0" colorama = "^0.4.3" diff --git a/test/unit/infra/media/__init__.py b/test/unit/infra/media/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/test/unit/infra/media/test_parser.py b/test/unit/infra/media/test_parser.py new file mode 100644 index 00000000..51abd636 --- /dev/null +++ b/test/unit/infra/media/test_parser.py @@ -0,0 +1,54 @@ +from collections import namedtuple +from test.util import TestCase +from unittest.mock import Mock + +from OpenCast.infra.media.parser import VideoParser, VideoParsingError +from vlc import MediaParsedStatus + +Stream = namedtuple("Stream", ["id", "type", "language"]) + + +class VideoParserTest(TestCase): + def setUp(self): + self.vlc = Mock() + self.parser = VideoParser(self.vlc) + + def test_parse_streams(self): + video_path = "/tmp/source.mp4" + media = Mock() + self.vlc.media_new.return_value = media + media.parse_with_options.return_value = 0 + media.get_parsed_status.return_value = MediaParsedStatus.done + media.is_parsed.return_value = 1 + + streams = [(f"id_{i}", f"type_{i}", f"language_{i}") for i in range(3)] + media.tracks_get.return_value = [Stream(*stream) for stream in streams] + self.assertEqual(streams, self.parser.parse_streams(video_path)) + + def test_parse_streams_failed(self): + video_path = "/tmp/source.mp4" + media = Mock() + self.vlc.media_new.return_value = media + status = MediaParsedStatus.failed + media.get_parsed_status.return_value = status + media.parse_with_options.return_value = -1 + with self.assertRaises(VideoParsingError) as ctx: + self.parser.parse_streams(video_path) + self.assertEqual( + f"Can't parse streams from '{video_path}', status='{status}'", + str(ctx.exception), + ) + + def test_parse_streams_timeout(self): + video_path = "/tmp/source.mp4" + media = Mock() + self.vlc.media_new.return_value = media + status = MediaParsedStatus.timeout + media.get_parsed_status.return_value = status + media.parse_with_options.return_value = 0 + with self.assertRaises(VideoParsingError) as ctx: + self.parser.parse_streams(video_path) + self.assertEqual( + f"Can't parse streams from '{video_path}', status='{status}'", + str(ctx.exception), + ) From e437cc2693ceb587c0079b713ecaf3e2380348aa Mon Sep 17 00:00:00 2001 From: Luc Sinet Date: Fri, 5 Jun 2020 14:57:11 +0200 Subject: [PATCH 20/29] replace-omxplayer #comment Add stream component to the video model. - Add a Pair of command / event for parsing the video streams. - Update the video workflow and tests. --- OpenCast/app/command/video.py | 5 +++++ OpenCast/app/workflow/video.py | 9 ++++++++- OpenCast/domain/event/video.py | 5 +++++ OpenCast/domain/model/video.py | 29 ++++++++++++++++++++++++++++ test/unit/app/workflow/test_video.py | 22 ++++++++++++++++++--- test/unit/domain/model/test_video.py | 6 ++++++ 6 files changed, 72 insertions(+), 4 deletions(-) diff --git a/OpenCast/app/command/video.py b/OpenCast/app/command/video.py index 865f0cb0..99bf1b7b 100644 --- a/OpenCast/app/command/video.py +++ b/OpenCast/app/command/video.py @@ -24,6 +24,11 @@ class RetrieveVideo(Command): output_directory: str +@command +class ParseVideo(Command): + pass + + @command class FetchVideoSubtitle(Command): language: str diff --git a/OpenCast/app/workflow/video.py b/OpenCast/app/workflow/video.py index 08f73eff..ae0b88c6 100644 --- a/OpenCast/app/workflow/video.py +++ b/OpenCast/app/workflow/video.py @@ -31,6 +31,7 @@ class States(Enum): CREATING = auto() IDENTIFYING = auto() RETRIEVING = auto() + PARSING = auto() FINALISING = auto() COMPLETED = auto() DELETING = auto() @@ -42,7 +43,8 @@ class States(Enum): ["_create", States.INITIAL, States.CREATING], ["_video_created", States.CREATING, States.IDENTIFYING], ["_video_identified", States.IDENTIFYING, States.RETRIEVING], - ["_video_retrieved", States.RETRIEVING, States.FINALISING], + ["_video_retrieved", States.RETRIEVING, States.PARSING], + ["_video_parsed", States.PARSING, States.FINALISING], ["_video_subtitle_fetched", States.FINALISING, States.COMPLETED], ["_operation_error", States.CREATING, States.ABORTED], @@ -81,6 +83,11 @@ def on_enter_RETRIEVING(self, _): config["downloader.output_directory"], ) + def on_enter_PARSING(self, _): + self._observe_dispatch( + VideoEvt.VideoParsed, Cmd.ParseVideo, self._video.id, + ) + def on_enter_FINALISING(self, _): self._observe_dispatch( VideoEvt.VideoSubtitleFetched, diff --git a/OpenCast/domain/event/video.py b/OpenCast/domain/event/video.py index da34e57b..5c6a0c3d 100644 --- a/OpenCast/domain/event/video.py +++ b/OpenCast/domain/event/video.py @@ -26,6 +26,11 @@ class VideoRetrieved(Event): path: Path +@dataclass +class VideoParsed(Event): + streams: list + + @dataclass class VideoSubtitleFetched(Event): subtitle: Path diff --git a/OpenCast/domain/model/video.py b/OpenCast/domain/model/video.py index 6ee0c80a..9c623b6a 100644 --- a/OpenCast/domain/model/video.py +++ b/OpenCast/domain/model/video.py @@ -1,10 +1,19 @@ +from dataclasses import dataclass from pathlib import Path +from typing import List from OpenCast.domain.event import video as Evt from .entity import Entity +@dataclass +class Stream: + index: int + type: str + language: str + + class Video(Entity): def __init__(self, id_, source, playlist_id): super(Video, self).__init__(id_) @@ -12,6 +21,7 @@ def __init__(self, id_, source, playlist_id): self._playlist_id = playlist_id self._title = None self._path = None + self._streams = None self._subtitle = None self._record(Evt.VideoCreated, self._source, self._playlist_id) @@ -36,6 +46,10 @@ def title(self): def playlist_id(self): return self._playlist_id + @property + def streams(self): + return self._streams + @property def subtitle(self): return self._subtitle @@ -54,6 +68,11 @@ def path(self, path: Path): self._path = path self._record(Evt.VideoRetrieved, self._path) + @streams.setter + def streams(self, streams: List[Stream]): + self._streams = streams + self._record(Evt.VideoParsed, self._streams) + @subtitle.setter def subtitle(self, subtitle: str): self._subtitle = subtitle @@ -62,5 +81,15 @@ def subtitle(self, subtitle: str): def from_disk(self): return Path(self._source).is_file() + def stream(self, type: str, language: str): + return next( + ( + stream + for stream in self._streams + if stream.type == type and stream.language == language + ), + None, + ) + def delete(self): self._record(Evt.VideoDeleted) diff --git a/test/unit/app/workflow/test_video.py b/test/unit/app/workflow/test_video.py index 275a9f44..9c104126 100644 --- a/test/unit/app/workflow/test_video.py +++ b/test/unit/app/workflow/test_video.py @@ -73,17 +73,33 @@ def test_retrieving_to_deleting(self): self.raise_error(self.workflow, cmd) self.assertTrue(self.workflow.is_DELETING()) - def test_retrieving_to_finalising(self): + def test_retrieving_to_parsing(self): event = Evt.VideoIdentified(None, self.video.id, "") self.workflow.to_RETRIEVING(event) cmd = self.expect_dispatch(Cmd.RetrieveVideo, self.video.id, "/tmp") self.raise_event( self.workflow, Evt.VideoRetrieved, cmd.id, self.video.id, "/tmp/video.mp4", ) + self.assertTrue(self.workflow.is_PARSING()) + + def test_parsing_to_deleting(self): + event = Evt.VideoRetrieved(None, self.video.id, "/tmp") + self.workflow.to_PARSING(event) + cmd = self.expect_dispatch(Cmd.ParseVideo, self.video.id) + self.raise_error(self.workflow, cmd) + self.assertTrue(self.workflow.is_DELETING()) + + def test_parsing_to_finalising(self): + event = Evt.VideoRetrieved(None, self.video.id, "/tmp") + self.workflow.to_PARSING(event) + cmd = self.expect_dispatch(Cmd.ParseVideo, self.video.id) + self.raise_event( + self.workflow, Evt.VideoParsed, cmd.id, self.video.id, {}, + ) self.assertTrue(self.workflow.is_FINALISING()) def test_finalising_to_deleting(self): - event = Evt.VideoRetrieved(None, self.video.id, "/tmp") + event = Evt.VideoParsed(None, self.video.id, {}) self.workflow.to_FINALISING(event) cmd = self.expect_dispatch( Cmd.FetchVideoSubtitle, self.video.id, config["subtitle.language"] @@ -92,7 +108,7 @@ def test_finalising_to_deleting(self): self.assertTrue(self.workflow.is_DELETING()) def test_finalising_to_completed(self): - event = Evt.VideoRetrieved(None, self.video.id, "/tmp") + event = Evt.VideoParsed(None, self.video.id, {}) self.workflow.to_FINALISING(event) cmd = self.expect_dispatch( Cmd.FetchVideoSubtitle, self.video.id, config["subtitle.language"] diff --git a/test/unit/domain/model/test_video.py b/test/unit/domain/model/test_video.py index 0435e579..00f53894 100644 --- a/test/unit/domain/model/test_video.py +++ b/test/unit/domain/model/test_video.py @@ -11,6 +11,7 @@ def test_construction(self): self.assertEqual("source", video.source) self.assertEqual(None, video.title) self.assertEqual(None, video.path) + self.assertEqual(None, video.streams) self.assertEqual(None, video.subtitle) self.expect_events(video, Evt.VideoCreated) @@ -30,6 +31,11 @@ def test_retrieve(self): video.path = "/tmp" self.expect_events(video, Evt.VideoRetrieved) + def test_parse(self): + video = self.make_video() + video.streams = {} + self.expect_events(video, Evt.VideoParsed) + def test_set_subtitles(self): video = self.make_video() video.subtitle = "/tmp/toto.srt" From ef998cfd0ff71ee6a341639d5d2111179f1338be Mon Sep 17 00:00:00 2001 From: Luc Sinet Date: Fri, 5 Jun 2020 17:24:18 +0200 Subject: [PATCH 21/29] replace-omxplayer #comment Add tests for source and subtitle services. - Remove subtitle fetching method. - Add source listing method. --- OpenCast/domain/service/source.py | 13 ++++++- OpenCast/domain/service/subtitle.py | 24 +----------- test/unit/domain/service/test_source.py | 47 +++++++++++++++++++++++ test/unit/domain/service/test_subtitle.py | 30 ++++++++++++++- 4 files changed, 89 insertions(+), 25 deletions(-) create mode 100644 test/unit/domain/service/test_source.py diff --git a/OpenCast/domain/service/source.py b/OpenCast/domain/service/source.py index 933190b4..43aa1510 100644 --- a/OpenCast/domain/service/source.py +++ b/OpenCast/domain/service/source.py @@ -1,9 +1,15 @@ from pathlib import Path +from typing import List + +import structlog +from OpenCast.domain.model.video import Stream class SourceService: - def __init__(self, downloader): + def __init__(self, downloader, video_parser): + self._logger = structlog.get_logger(__name__) self._downloader = downloader + self._video_parser = video_parser def is_playlist(self, source): return "/playlist" in source @@ -15,3 +21,8 @@ def pick_stream_metadata(self, video): if video.from_disk(): return {"title": str(Path(video.source).name)} return self._downloader.pick_stream_metadata(video.source, ["title"]) + + def list_streams(self, video) -> List[Stream]: + video_path = str(video.path) + streams = self._video_parser.parse_streams(video_path) + return [Stream(*stream) for stream in streams] diff --git a/OpenCast/domain/service/subtitle.py b/OpenCast/domain/service/subtitle.py index 74a43829..838bb81c 100644 --- a/OpenCast/domain/service/subtitle.py +++ b/OpenCast/domain/service/subtitle.py @@ -4,14 +4,11 @@ class SubtitleService: - def __init__(self, ffmpeg_wrapper, downloader): + def __init__(self, downloader): self._logger = structlog.get_logger(__name__) - self._ffmpeg_wrapper = ffmpeg_wrapper self._downloader = downloader - def fetch_subtitle( - self, video, language: str, search_source=True, search_online=True - ) -> Path: + def fetch_subtitle(self, video, language: str, search_online=True) -> Path: subtitle = self._load_from_disk(video.path, language) if subtitle is not None: return subtitle @@ -35,23 +32,6 @@ def _load_from_disk(self, video_path: Path, language: str) -> str: if Path(subtitle) in srtFiles: self._logger.debug("Found srt file", subtitle=subtitle) return subtitle - - # Extract file metadata - # Find subtitle with matching language - self._logger.debug("Searching softcoded subtitles", subtitle=subtitle) - metadata = self._ffmpeg_wrapper.probe(video_path) - for stream in metadata["streams"]: - self._logger.debug( - f"Channel #{stream['index']}", - type=stream["codec_type"], - name=stream["codec_long_name"], - ) - if ( - stream["codec_type"] == "subtitle" - and stream["tags"]["language"] == language - ): - self._logger.debug(f"Match: {subtitle}") - return subtitle return None def _download_from_source( diff --git a/test/unit/domain/service/test_source.py b/test/unit/domain/service/test_source.py new file mode 100644 index 00000000..e1647048 --- /dev/null +++ b/test/unit/domain/service/test_source.py @@ -0,0 +1,47 @@ +from pathlib import Path +from test.util import TestCase +from unittest.mock import Mock + +from OpenCast.domain.model.video import Stream +from OpenCast.domain.service.source import SourceService + + +class SourceServiceTest(TestCase): + def setUp(self): + self.downloader = Mock() + self.video_parser = Mock() + self.service = SourceService(self.downloader, self.video_parser) + + def test_is_playlist(self): + self.assertTrue( + self.service.is_playlist("https://www.youtube.com/playlist?list=id") + ) + self.assertFalse( + self.service.is_playlist( + "https://www.youtube.com/watch?v=id&list=id&index=2" + ) + ) + + def test_pick_stream_metadata_from_disk(self): + video = Mock() + video.from_disk.return_value = True + video.source = "/tmp/toto.mp4" + + metadata = self.service.pick_stream_metadata(video) + expected = {"title": "toto"} + self.assertEqual(expected, metadata) + + def test_list_streams(self): + video = Mock() + video.Path = Path("/tmp/toto.mp4") + + streams = [ + (0, "video", "video_lang"), + (1, "audio", "audio_lang"), + (2, "subtitle", "subtitle_lang"), + ] + self.video_parser.parse_streams.return_value = streams + + expected = [Stream(*stream) for stream in streams] + stream_list = self.service.list_streams(video) + self.assertEqual(expected, stream_list) diff --git a/test/unit/domain/service/test_subtitle.py b/test/unit/domain/service/test_subtitle.py index 35334469..9035e858 100644 --- a/test/unit/domain/service/test_subtitle.py +++ b/test/unit/domain/service/test_subtitle.py @@ -1,11 +1,37 @@ +from pathlib import Path from test.util import TestCase +from unittest.mock import MagicMock, Mock +from OpenCast.config import config from OpenCast.domain.service.subtitle import SubtitleService class SubtitleServiceTest(TestCase): def setUp(self): - self._service = SubtitleService(None, None) + self.downloader = Mock() + self.service = SubtitleService(self.downloader) def test_load_from_disk(self): - pass + path_mock = MagicMock() + language = config["subtitle.language"] + + parent_mock = Mock() + path_mock.parents.__getitem__.return_value = parent_mock + exp_subtitle = Path("/tmp/source.srt") + path_mock.with_suffix.return_value = exp_subtitle + + parent_mock.glob.return_value = [exp_subtitle] + + subtitle = self.service.load_from_disk(path_mock, language) + self.assertEqual(str(exp_subtitle), subtitle) + + def test_download_from_source(self): + video_source = "http://someurl/id" + video_path = Mock() + language = config["subtitle.language"] + + source_subtitle = "/tmp/source.vtt" + self.downloader.download_subtitle.return_value = source_subtitle + + subtitle = self.service.download_from_source(video_source, video_path, language) + self.assertEqual(Path(source_subtitle), subtitle) From b4b392c82b646d53eb4f7476c8799923a6f02fb1 Mon Sep 17 00:00:00 2001 From: Luc Sinet Date: Fri, 5 Jun 2020 17:25:12 +0200 Subject: [PATCH 22/29] replace-omxplayer #comment Append test to test selector only if not already present. --- OpenCast.sh | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/OpenCast.sh b/OpenCast.sh index c923248e..b4161770 100755 --- a/OpenCast.sh +++ b/OpenCast.sh @@ -80,7 +80,12 @@ function test() { if [ -z "$1" ]; then run_in_env python -m unittest discover -v else - run_in_env python -m unittest "$TEST_DIR.$1" + local selector="$1" + + if [[ "$selector" != "$TEST_DIR"* ]]; then + selector="$TEST_DIR.$selector" + fi + run_in_env python -m unittest "$selector" fi } From 2a1565de0ac6dd3346ff707a6c9963e651698b96 Mon Sep 17 00:00:00 2001 From: Luc Sinet Date: Fri, 5 Jun 2020 17:27:45 +0200 Subject: [PATCH 23/29] replace-omxplayer #comment Set streams to empty list instead of None. - This ensures that the stream method won't try to iterate over None + It is a more coherent definition. --- OpenCast/domain/model/video.py | 2 +- test/unit/domain/model/test_video.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/OpenCast/domain/model/video.py b/OpenCast/domain/model/video.py index 9c623b6a..5d6732fd 100644 --- a/OpenCast/domain/model/video.py +++ b/OpenCast/domain/model/video.py @@ -21,7 +21,7 @@ def __init__(self, id_, source, playlist_id): self._playlist_id = playlist_id self._title = None self._path = None - self._streams = None + self._streams = [] self._subtitle = None self._record(Evt.VideoCreated, self._source, self._playlist_id) diff --git a/test/unit/domain/model/test_video.py b/test/unit/domain/model/test_video.py index 00f53894..3e64bf9f 100644 --- a/test/unit/domain/model/test_video.py +++ b/test/unit/domain/model/test_video.py @@ -11,7 +11,7 @@ def test_construction(self): self.assertEqual("source", video.source) self.assertEqual(None, video.title) self.assertEqual(None, video.path) - self.assertEqual(None, video.streams) + self.assertEqual([], video.streams) self.assertEqual(None, video.subtitle) self.expect_events(video, Evt.VideoCreated) From bd5c4dafae22bed89a6faea7997e45780d959442 Mon Sep 17 00:00:00 2001 From: Luc Sinet Date: Fri, 5 Jun 2020 17:34:22 +0200 Subject: [PATCH 24/29] replace-omxplayer #comment Collection of minor improvements. --- OpenCast/domain/service/source.py | 2 +- OpenCast/domain/service/subtitle.py | 15 +++++++-------- OpenCast/infra/media/parser.py | 2 +- test/unit/infra/media/test_parser.py | 13 ++++++++++--- 4 files changed, 19 insertions(+), 13 deletions(-) diff --git a/OpenCast/domain/service/source.py b/OpenCast/domain/service/source.py index 43aa1510..0800131c 100644 --- a/OpenCast/domain/service/source.py +++ b/OpenCast/domain/service/source.py @@ -19,7 +19,7 @@ def unfold(self, source): def pick_stream_metadata(self, video): if video.from_disk(): - return {"title": str(Path(video.source).name)} + return {"title": Path(video.source).stem} return self._downloader.pick_stream_metadata(video.source, ["title"]) def list_streams(self, video) -> List[Stream]: diff --git a/OpenCast/domain/service/subtitle.py b/OpenCast/domain/service/subtitle.py index 838bb81c..d151617c 100644 --- a/OpenCast/domain/service/subtitle.py +++ b/OpenCast/domain/service/subtitle.py @@ -9,12 +9,12 @@ def __init__(self, downloader): self._downloader = downloader def fetch_subtitle(self, video, language: str, search_online=True) -> Path: - subtitle = self._load_from_disk(video.path, language) + subtitle = self.load_from_disk(video.path, language) if subtitle is not None: return subtitle if not video.from_disk(): - subtitle = self._download_from_source(video.source, video.path, language) + subtitle = self.download_from_source(video.source, video.path, language) if subtitle is not None: return subtitle @@ -23,22 +23,21 @@ def fetch_subtitle(self, video, language: str, search_online=True) -> Path: return None - def _load_from_disk(self, video_path: Path, language: str) -> str: + def load_from_disk(self, video_path: Path, language: str) -> Path: parent_path = video_path.parents[0] subtitle = str(video_path.with_suffix(".srt")) - # Find the matching subtitle from a .srt file - srtFiles = list(parent_path.glob("*.srt")) + srtFiles = parent_path.glob("*.srt") if Path(subtitle) in srtFiles: self._logger.debug("Found srt file", subtitle=subtitle) return subtitle return None - def _download_from_source( + def download_from_source( self, video_source: str, video_path: Path, language: str - ) -> str: + ) -> Path: dest = str(video_path.with_suffix("")) subtitle = self._downloader.download_subtitle( video_source, dest, language, ["vtt"] ) - return subtitle + return Path(subtitle) diff --git a/OpenCast/infra/media/parser.py b/OpenCast/infra/media/parser.py index 5fcc91ef..05032eee 100644 --- a/OpenCast/infra/media/parser.py +++ b/OpenCast/infra/media/parser.py @@ -49,7 +49,7 @@ def raise_on_error(): ( stream.id, type_to_code.get(stream.type, "unknown"), - stream.language.decode("UTF-8"), + None if stream.language is None else stream.language.decode("UTF-8"), ) for stream in streams ] diff --git a/test/unit/infra/media/test_parser.py b/test/unit/infra/media/test_parser.py index 51abd636..00f1dc32 100644 --- a/test/unit/infra/media/test_parser.py +++ b/test/unit/infra/media/test_parser.py @@ -3,7 +3,7 @@ from unittest.mock import Mock from OpenCast.infra.media.parser import VideoParser, VideoParsingError -from vlc import MediaParsedStatus +from vlc import MediaParsedStatus, TrackType Stream = namedtuple("Stream", ["id", "type", "language"]) @@ -21,9 +21,16 @@ def test_parse_streams(self): media.get_parsed_status.return_value = MediaParsedStatus.done media.is_parsed.return_value = 1 - streams = [(f"id_{i}", f"type_{i}", f"language_{i}") for i in range(3)] + input_languages = [b"eng", None, b"\xE2\x82\xAC"] + output_languages = ["eng", None, "€"] + input_types = [TrackType.audio, TrackType.video, TrackType.ext] + output_types = ["audio", "video", "subtitle"] + + streams = [(i, input_types[i], input_languages[i]) for i in range(3)] + expected = [(i, output_types[i], output_languages[i]) for i in range(3)] + media.tracks_get.return_value = [Stream(*stream) for stream in streams] - self.assertEqual(streams, self.parser.parse_streams(video_path)) + self.assertEqual(expected, self.parser.parse_streams(video_path)) def test_parse_streams_failed(self): video_path = "/tmp/source.mp4" From f69598058ee4b4daf7eecd524c89b5e763e4cd44 Mon Sep 17 00:00:00 2001 From: Luc Sinet Date: Fri, 5 Jun 2020 17:36:55 +0200 Subject: [PATCH 25/29] replace-omxplayer #comment Move the downloader in media. --- OpenCast/app/controller/player_monitor.py | 4 +++- OpenCast/app/service/module.py | 2 +- OpenCast/app/service/video.py | 10 +++++----- OpenCast/infra/io/error.py | 2 -- OpenCast/infra/io/factory.py | 11 ----------- OpenCast/infra/{io => media}/download_logger.py | 0 OpenCast/infra/{io => media}/downloader.py | 0 OpenCast/infra/media/factory.py | 10 ++++++++-- 8 files changed, 17 insertions(+), 22 deletions(-) delete mode 100644 OpenCast/infra/io/error.py rename OpenCast/infra/{io => media}/download_logger.py (100%) rename OpenCast/infra/{io => media}/downloader.py (100%) diff --git a/OpenCast/app/controller/player_monitor.py b/OpenCast/app/controller/player_monitor.py index c377bfc2..de72e180 100755 --- a/OpenCast/app/controller/player_monitor.py +++ b/OpenCast/app/controller/player_monitor.py @@ -19,8 +19,10 @@ class PlayerMonitController(Controller): def __init__(self, app_facade, infra_facade, data_facade, service_factory): super(PlayerMonitController, self).__init__(app_facade) + media_factory = infra_facade.media_factory self._source_service = service_factory.make_source_service( - infra_facade.io_factory.make_downloader(app_facade.evt_dispatcher) + media_factory.make_downloader(app_facade.evt_dispatcher), + media_factory.make_video_parser(), ) self._player_repo = data_facade.player_repo self._video_repo = data_facade.video_repo diff --git a/OpenCast/app/service/module.py b/OpenCast/app/service/module.py index 0f2f403e..fa4572c9 100644 --- a/OpenCast/app/service/module.py +++ b/OpenCast/app/service/module.py @@ -8,5 +8,5 @@ def __init__(self, app_facade, infra_facade, data_facade, service_factory): app_facade, data_facade, infra_facade.media_factory ) self._video_service = VideoService( - app_facade, service_factory, data_facade, infra_facade.io_factory + app_facade, service_factory, data_facade, infra_facade.media_factory ) diff --git a/OpenCast/app/service/video.py b/OpenCast/app/service/video.py index 33770ee3..44a0c81d 100644 --- a/OpenCast/app/service/video.py +++ b/OpenCast/app/service/video.py @@ -9,15 +9,15 @@ class VideoService(Service): - def __init__(self, app_facade, service_factory, data_facade, io_factory): + def __init__(self, app_facade, service_factory, data_facade, media_factory): logger = structlog.get_logger(__name__) super(VideoService, self).__init__(app_facade, logger, self, video_cmds) self._video_repo = data_facade.video_repo - self._downloader = io_factory.make_downloader(app_facade.evt_dispatcher) - self._source_service = service_factory.make_source_service(self._downloader) - self._subtitle_service = service_factory.make_subtitle_service( - io_factory.make_ffmpeg_wrapper(), self._downloader + self._downloader = media_factory.make_downloader(app_facade.evt_dispatcher) + self._source_service = service_factory.make_source_service( + self._downloader, media_factory.make_video_parser() ) + self._subtitle_service = service_factory.make_subtitle_service(self._downloader) # Command handler interface implementation def _create_video(self, cmd): diff --git a/OpenCast/infra/io/error.py b/OpenCast/infra/io/error.py deleted file mode 100644 index 286ccd36..00000000 --- a/OpenCast/infra/io/error.py +++ /dev/null @@ -1,2 +0,0 @@ -class DownloadError(Exception): - pass diff --git a/OpenCast/infra/io/factory.py b/OpenCast/infra/io/factory.py index 2b0d1cd2..6fcb68fa 100644 --- a/OpenCast/infra/io/factory.py +++ b/OpenCast/infra/io/factory.py @@ -1,17 +1,6 @@ -from .ffmpeg_wrapper import FFmpegWrapper from .server import Server -from .downloader import Downloader class IoFactory: - def __init__(self, downloader_executor): - self._downloader_executor = downloader_executor - - def make_ffmpeg_wrapper(self, *args): - return FFmpegWrapper(*args) - def make_server(self, *args): return Server(*args) - - def make_downloader(self, *args): - return Downloader(self._downloader_executor, *args) diff --git a/OpenCast/infra/io/download_logger.py b/OpenCast/infra/media/download_logger.py similarity index 100% rename from OpenCast/infra/io/download_logger.py rename to OpenCast/infra/media/download_logger.py diff --git a/OpenCast/infra/io/downloader.py b/OpenCast/infra/media/downloader.py similarity index 100% rename from OpenCast/infra/io/downloader.py rename to OpenCast/infra/media/downloader.py diff --git a/OpenCast/infra/media/factory.py b/OpenCast/infra/media/factory.py index 73894d68..e68ea3eb 100644 --- a/OpenCast/infra/media/factory.py +++ b/OpenCast/infra/media/factory.py @@ -1,5 +1,5 @@ -from pathlib import Path - +from .downloader import Downloader +from .parser import VideoParser from .player_wrapper import PlayerWrapper @@ -10,3 +10,9 @@ def __init__(self, vlc_instance, downloader_executor): def make_player(self, *args): return PlayerWrapper(self._vlc, *args) + + def make_downloader(self, *args): + return Downloader(self._downloader_executor, *args) + + def make_video_parser(self, *args): + return VideoParser(self._vlc, *args) From b72a60e09f917d44a361f1545646b310eb0b5c79 Mon Sep 17 00:00:00 2001 From: Luc Sinet Date: Fri, 5 Jun 2020 17:38:26 +0200 Subject: [PATCH 26/29] replace-omxplayer #comment Implement the parse command handler. --- OpenCast/app/service/video.py | 9 ++++ test/integration/app/service/test_video.py | 61 ++++++++++++++-------- 2 files changed, 48 insertions(+), 22 deletions(-) diff --git a/OpenCast/app/service/video.py b/OpenCast/app/service/video.py index 44a0c81d..950a45c8 100644 --- a/OpenCast/app/service/video.py +++ b/OpenCast/app/service/video.py @@ -75,6 +75,15 @@ def abort_operation(evt): ) self._downloader.download_video(cmd.id, video.source, str(video.path)) + def _parse_video(self, cmd): + def impl(ctx): + video = self._video_repo.get(cmd.model_id) + streams = self._source_service.list_streams(video) + video.streams = streams + ctx.update(video) + + self._start_transaction(self._video_repo, cmd.id, impl) + def _fetch_video_subtitle(self, cmd): def impl(ctx): video = self._video_repo.get(cmd.model_id) diff --git a/test/integration/app/service/test_video.py b/test/integration/app/service/test_video.py index 2d427017..d45855fa 100644 --- a/test/integration/app/service/test_video.py +++ b/test/integration/app/service/test_video.py @@ -1,12 +1,12 @@ from pathlib import Path -from unittest.mock import Mock +from unittest.mock import MagicMock, Mock from OpenCast.app.command import video as Cmd from OpenCast.app.service.error import OperationError from OpenCast.app.service.video import VideoService from OpenCast.config import config from OpenCast.domain.event import video as Evt -from OpenCast.domain.model.video import Video +from OpenCast.domain.model.video import Stream, Video from OpenCast.domain.service.identity import IdentityService from OpenCast.infra.event.downloader import DownloadError, DownloadSuccess @@ -18,13 +18,13 @@ def setUp(self): super(VideoServiceTest, self).setUp() self.downloader = Mock() - self.ffmpeg_wrapper = Mock() - self.io_factory = self.infra_facade.io_factory - self.io_factory.make_downloader.return_value = self.downloader - self.io_factory.make_ffmpeg_wrapper.return_value = self.ffmpeg_wrapper + self.video_parser = Mock() + self.media_factory = self.infra_facade.media_factory + self.media_factory.make_downloader.return_value = self.downloader + self.media_factory.make_video_parser.return_value = self.video_parser self.service = VideoService( - self.app_facade, self.service_factory, self.data_facade, self.io_factory + self.app_facade, self.service_factory, self.data_facade, self.media_factory ) self.video_repo = self.data_facade.video_repo @@ -98,26 +98,43 @@ def dispatch_error(op_id, *args): Cmd.RetrieveVideo, video_id, output_dir ) - def test_fetch_video_subtitle(self): + def test_parse_video(self): self.data_producer.video("source", None, path=Path("/tmp/source.mp4")).populate( self.data_facade ) - subtitle = "/tmp/source.srt" - subtitle_language = config["subtitle.language"] - self.ffmpeg_wrapper.probe.return_value = { - "streams": [ - { - "index": 1, - "codec_type": "subtitle", - "codec_long_name": "subtitle", - "tags": {"language": subtitle_language}, - } - ] - } - self.ffmpeg_wrapper.extract_stream.return_value = subtitle + streams = [ + (0, "video", None), + (1, "audio", None), + (2, "subtitle", "subtitle_lang"), + ] + self.video_parser.parse_streams.return_value = streams + + expected = [Stream(*stream) for stream in streams] + video_id = IdentityService.id_video("source") + self.evt_expecter.expect(Evt.VideoParsed, expected).from_( + Cmd.ParseVideo, video_id + ) + + def test_fetch_video_subtitle(self): + path_mock = MagicMock() + self.data_producer.video("source", None, path=path_mock).populate( + self.data_facade + ) + + # Load from disk + disk_subtitle = "/tmp/source.srt" + path_mock.with_suffix.return_value = disk_subtitle + parent_mock = Mock() + path_mock.parents.__getitem__.return_value = parent_mock + parent_mock.glob.return_value = [] + + # Download from source + source_subtitle = "/tmp/source.vtt" + self.downloader.download_subtitle.return_value = source_subtitle video_id = IdentityService.id_video("source") - self.evt_expecter.expect(Evt.VideoSubtitleFetched, subtitle).from_( + subtitle_language = config["subtitle.language"] + self.evt_expecter.expect(Evt.VideoSubtitleFetched, Path(source_subtitle)).from_( Cmd.FetchVideoSubtitle, video_id, subtitle_language ) From 89f0355d965fb67f9e19563009720106bc836092 Mon Sep 17 00:00:00 2001 From: Luc Sinet Date: Fri, 5 Jun 2020 17:42:42 +0200 Subject: [PATCH 27/29] replace-omxplayer #comment Make the language config entry 3 chars and set the subtitles on by default. --- OpenCast/config.py | 2 +- OpenCast/domain/model/player.py | 2 +- config.yml | 2 +- test/unit/domain/model/test_player.py | 2 +- 4 files changed, 4 insertions(+), 4 deletions(-) diff --git a/OpenCast/config.py b/OpenCast/config.py index 50597a57..e8bf5093 100644 --- a/OpenCast/config.py +++ b/OpenCast/config.py @@ -179,7 +179,7 @@ def _override_from_env(self, content: dict, env: dict, key: list): "max_concurrency": 3 }, "subtitle": { - "language": "en" + "language": "eng" } }, check_env=True) # fmt: on diff --git a/OpenCast/domain/model/player.py b/OpenCast/domain/model/player.py index fd4c6b6a..48ea29a0 100644 --- a/OpenCast/domain/model/player.py +++ b/OpenCast/domain/model/player.py @@ -18,7 +18,7 @@ def __init__(self, id_): self._state = PlayerState.STOPPED self._queue = [] self._index = 0 - self._sub_state = False + self._sub_state = True self._sub_delay = 0 self._volume = 70 diff --git a/config.yml b/config.yml index 0c3bd121..190f2dcf 100644 --- a/config.yml +++ b/config.yml @@ -24,4 +24,4 @@ downloader: subtitle: # The default language for subtitles. - language: en + language: eng diff --git a/test/unit/domain/model/test_player.py b/test/unit/domain/model/test_player.py index d6c5785e..1e158dc8 100644 --- a/test/unit/domain/model/test_player.py +++ b/test/unit/domain/model/test_player.py @@ -21,7 +21,7 @@ def make_video(self, source="source_1"): def test_construction(self): self.assertEqual(70, self.player.volume) - self.assertFalse(self.player.subtitle_state) + self.assertTrue(self.player.subtitle_state) self.assertEqual(0, self.player.subtitle_delay) self.assertEqual(PlayerState.STOPPED, self.player.state) From 0aa593f6720a43bc52200a474a6807a91352bf71 Mon Sep 17 00:00:00 2001 From: Luc Sinet Date: Fri, 5 Jun 2020 17:45:18 +0200 Subject: [PATCH 28/29] replace-omxplayer #comment Add subtitle selection capability to the player wrapper. - Set the subtitle with the prefered language when available. --- OpenCast/app/service/player.py | 6 ++++++ OpenCast/infra/media/player_wrapper.py | 26 +++++++++++++++++++++----- 2 files changed, 27 insertions(+), 5 deletions(-) diff --git a/OpenCast/app/service/player.py b/OpenCast/app/service/player.py index 83f0d7a7..5ed66062 100644 --- a/OpenCast/app/service/player.py +++ b/OpenCast/app/service/player.py @@ -1,6 +1,7 @@ import structlog from OpenCast.app.command import player as player_cmds from OpenCast.app.error import CommandFailure +from OpenCast.config import config from OpenCast.domain.model.player_state import PlayerState from .service import Service @@ -104,6 +105,11 @@ def play_video(model): model.play(video) self._player.play(video.id, str(video.path)) + player = self._player_model() + if player.subtitle_state is True: + sub_stream = video.stream("subtitle", config["subtitle.language"]) + if sub_stream is not None: + self._player.select_subtitle_stream(sub_stream.index) self._update(cmd_id, play_video) def _queue_video_impl(self, cmd_id, video): diff --git a/OpenCast/infra/media/player_wrapper.py b/OpenCast/infra/media/player_wrapper.py index 28377325..93297872 100644 --- a/OpenCast/infra/media/player_wrapper.py +++ b/OpenCast/infra/media/player_wrapper.py @@ -1,9 +1,11 @@ -from threading import Lock +from threading import Condition from uuid import UUID import OpenCast.infra.event.player as e import structlog -import vlc +from vlc import EventType + +from .error import PlayerError class PlayerWrapper: @@ -16,9 +18,6 @@ def __init__(self, vlc_instance, evt_dispatcher): self._player = self._instance.media_player_new() self._id_to_media = {} - self._lock = Lock() - self._stop_operation_id = None - player_events = self._player.event_manager() player_events.event_attach(EventType.MediaPlayerEndReached, self._on_media_end) @@ -39,6 +38,23 @@ def pause(self): def unpause(self): self._player.pause() + def select_subtitle_stream(self, index: int): + media = self._player.get_media() + if media is None: + raise PlayerError("the player is not started") + + def is_playing(_, cv): + with cv: + cv.notify() + + cv = Condition() + self._player.event_manager().event_attach( + EventType.MediaPlayerPlaying, is_playing, cv + ) + with cv: + cv.wait_for(self._player.is_playing) + self._player.video_set_spu(index) + def toggle_subtitle(self): self._player.toggle_teletext() From 57f7ea620d8a78b6d7985a6f2d07fb32c350797a Mon Sep 17 00:00:00 2001 From: Luc Sinet Date: Fri, 5 Jun 2020 20:37:00 +0200 Subject: [PATCH 29/29] replace-omxplayer #comment Ensure using the right language code when downloading subtitles. --- OpenCast/infra/media/downloader.py | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/OpenCast/infra/media/downloader.py b/OpenCast/infra/media/downloader.py index cf84e9d8..f92e6d34 100644 --- a/OpenCast/infra/media/downloader.py +++ b/OpenCast/infra/media/downloader.py @@ -1,6 +1,7 @@ from typing import List -import youtube_dl +from youtube_dl import YoutubeDL +from youtube_dl.utils import ISO639Utils import structlog from OpenCast.infra.event.downloader import DownloadError, DownloadSuccess @@ -29,7 +30,7 @@ def impl(): "quiet": True, "progress_hooks": [self._dl_logger.log_download_progress], } - ydl = youtube_dl.YoutubeDL(options) + ydl = YoutubeDL(options) with ydl: # Download the video try: ydl.download([source]) @@ -49,6 +50,7 @@ def impl(): def download_subtitle(self, url: str, dest: str, lang: str, exts: List[str]): self._logger.debug("Downloading subtitle", subtitle=dest, lang=lang) + lang = ISO639Utils.long2short(lang) for ext in exts: options = { "skip_download": True, @@ -59,7 +61,7 @@ def download_subtitle(self, url: str, dest: str, lang: str, exts: List[str]): "progress_hooks": [self._dl_logger.log_download_progress], "quiet": True, } - ydl = youtube_dl.YoutubeDL(options) + ydl = YoutubeDL(options) with ydl: try: ydl.download([url]) @@ -103,7 +105,7 @@ def _download_stream_metadata(self, url, options): "progress_hooks": [self._dl_logger.log_download_progress], } ) - ydl = youtube_dl.YoutubeDL(options) + ydl = YoutubeDL(options) with ydl: try: return ydl.extract_info(url, download=False)