Add command for rendering EPUB files

fimfarchive/commands/__init__.py

@@ -25,12 +25,14 @@
 from .base import Command
 from .build import BuildCommand
 from .root import RootCommand
+from .render import RenderCommand
 from .update import UpdateCommand
 
 
 __all__ = (
     'Command',
     'RootCommand',
     'BuildCommand',
+    'RenderCommand',
     'UpdateCommand',
 )

fimfarchive/commands/render.py

@@ -0,0 +1,113 @@
+"""
+Render command.
+"""
+
+
+#
+# Fimfarchive, preserves stories from Fimfiction.
+# Copyright (C) 2019  Joakim Soderlund
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program.  If not, see <http://www.gnu.org/licenses/>.
+#
+
+
+from argparse import ArgumentParser, Namespace
+from pathlib import Path
+
+import arrow
+
+from fimfarchive.signals import SignalReceiver
+from fimfarchive.tasks import RenderTask
+from fimfarchive.utils import tqdm
+
+from .base import Command
+
+
+__all__ = (
+    'RenderCommand',
+)
+
+
+class RenderPrinter(SignalReceiver):
+
+    def on_enter(self, sender, keys, workers, spec):
+        self.entered = arrow.utcnow()
+        print(f"\nStarted: {self.entered}")
+        print(f"Directory: {spec.worktree}")
+        print(f"Stories: {len(keys)}")
+        print(f"Workers: {workers}\n")
+        self.tqdm = tqdm(total=len(keys))
+
+    def on_success(self, sender, key):
+        self.tqdm.update()
+
+    def on_failure(self, sender, key, error):
+        self.tqdm.write(f"[{key:6}] {error}")
+        self.tqdm.update()
+
+    def on_exit(self, sender, converted, remaining):
+        self.tqdm.close()
+        self.exited = arrow.utcnow()
+        print(f"\nDone: {self.exited}")
+        print(f"Duration: {self.exited - self.entered}")
+        print(f"Converted: {len(converted)}")
+        print(f"Remaining: {len(remaining)}\n")
+
+
+class RenderCommand(Command):
+    """
+    Renders updates as EPUB files.
+    """
+
+    @property
+    def parser(self) -> ArgumentParser:
+        """
+        Returns a command line arguments parser.
+        """
+        parser = ArgumentParser(
+            prog='',
+            description=self.__doc__,
+        )
+
+        parser.add_argument(
+            '--worktree',
+            help="Working directory for the archive",
+            metavar='PATH',
+            default='worktree',
+        )
+
+        return parser
+
+    def configure(self, opts: Namespace) -> RenderTask:
+        """
+        Returns a configured task instance.
+
+        Args:
+            opts: Parsed command line arguments.
+        """
+        worktree = Path(opts.worktree).resolve()
+
+        if not worktree.is_dir():
+            self.parser.error(f"No such directory: {worktree}")
+
+        return RenderTask(str(worktree))
+
+    def __call__(self, *args):
+        opts = self.parser.parse_args(args)
+        task = self.configure(opts)
+
+        with RenderPrinter(task):
+            task.run()
+
+        return 0

fimfarchive/commands/root.py

@@ -26,6 +26,7 @@
 
 from .base import Command
 from .build import BuildCommand
+from .render import RenderCommand
 from .update import UpdateCommand
 
 
@@ -40,6 +41,7 @@ class RootCommand(Command):
     """
     commands: Dict[str, Type[Command]] = {
         'build': BuildCommand,
+        'render': RenderCommand,
         'update': UpdateCommand,
     }
 

fimfarchive/tasks/__init__.py

@@ -23,10 +23,12 @@
 
 
 from .build import BuildTask
+from .render import RenderTask
 from .update import UpdateTask
 
 
 __all__ = (
     'BuildTask',
+    'RenderTask',
     'UpdateTask',
 )

fimfarchive/tasks/render.py

@@ -0,0 +1,171 @@
+"""
+Render task.
+"""
+
+
+#
+# Fimfarchive, preserves stories from Fimfiction.
+# Copyright (C) 2019  Joakim Soderlund
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program.  If not, see <http://www.gnu.org/licenses/>.
+#
+
+
+from multiprocessing import Pool
+from os import cpu_count
+from pathlib import Path
+from typing import List, Optional, Tuple
+
+from fimfarchive.converters import JsonFpubConverter, FpubEpubConverter
+from fimfarchive.fetchers import DirectoryFetcher
+from fimfarchive.flavors import DataFormat, MetaFormat
+from fimfarchive.mappers import MetaFormatMapper
+from fimfarchive.signals import Signal, SignalSender
+from fimfarchive.stories import Story
+from fimfarchive.writers import DirectoryWriter
+
+
+__all__ = (
+    'RenderTask',
+)
+
+
+WORKERS = 4
+WORKTREE = 'worktree'
+
+
+class PathSpec:
+    def __init__(self, worktree: str) -> None:
+        self.worktree = Path(worktree)
+        self.source = self.worktree / 'update'
+        self.target = self.worktree / 'render'
+        self.meta = self.source / 'meta'
+        self.json = self.source / 'json'
+        self.epub = self.target / 'epub'
+        self.logs = self.target / 'logs'
+
+    def verify_dir(self, path: Path) -> None:
+        if not path.is_dir():
+            raise ValueError(f"Missing dir: {path}")
+
+    def create_dir(self, path: Path) -> None:
+        path.mkdir(mode=0o755, parents=True, exist_ok=True)
+
+    def prepare(self) -> None:
+        self.verify_dir(self.meta)
+        self.verify_dir(self.json)
+        self.create_dir(self.epub)
+        self.create_dir(self.logs)
+
+
+class Executor:
+    initialized: bool = False
+
+    def __init__(self, worktree: str) -> None:
+        self.worktree = worktree
+
+    def initialize(self) -> None:
+        path = PathSpec(self.worktree)
+
+        self.fetcher = DirectoryFetcher(
+            meta_path=path.meta,
+            data_path=path.json,
+            flavors=[DataFormat.JSON],
+        )
+
+        self.writer = DirectoryWriter(
+            data_path=str(path.epub),
+            make_dirs=False,
+        )
+
+        self.to_fpub = JsonFpubConverter()
+        self.to_epub = FpubEpubConverter(str(path.logs))
+        self.get_meta_format = MetaFormatMapper()
+        self.initialized = True
+
+    def fetch(self, key: int) -> Story:
+        story = self.fetcher.fetch(key)
+
+        if MetaFormat.BETA in story.flavors:
+            raise ValueError("Flavor should not be static: {MetaFormat.BETA}")
+
+        if self.get_meta_format(story) != MetaFormat.BETA:
+            raise ValueError("Flavor could not be detected: {MetaFormat.BETA}")
+
+        story.flavors.add(MetaFormat.BETA)
+
+        return story
+
+    def apply(self, key: int) -> None:
+        json = self.fetch(key)
+        fpub = self.to_fpub(json)
+        epub = self.to_epub(fpub)
+        self.writer.write(epub)
+
+    def __call__(self, key: int) -> Tuple[int, Optional[str]]:
+        if not self.initialized:
+            self.initialize()
+
+        try:
+            self.apply(key)
+        except Exception as e:
+            return key, f"{type(e).__name__}: {e}"
+        else:
+            return key, None
+
+
+class RenderTask(SignalSender):
+    on_enter = Signal('keys', 'workers', 'spec')
+    on_exit = Signal('converted', 'remaining')
+    on_success = Signal('key')
+    on_failure = Signal('key', 'error')
+
+    def __init__(self, worktree: str = WORKTREE) -> None:
+        """
+        Constructor.
+        """
+        super().__init__()
+        self.worktree = worktree
+
+    def subtasks(self, spec: PathSpec) -> List[int]:
+        sources = {int(path.name) for path in spec.json.iterdir()}
+        targets = {int(path.name) for path in spec.epub.iterdir()}
+
+        return sorted(sources - targets)
+
+    def run(self) -> None:
+        spec = PathSpec(self.worktree)
+
+        spec.prepare()
+        keys = self.subtasks(spec)
+        func = Executor(self.worktree)
+
+        workers = cpu_count() or WORKERS
+        converted: List[int] = list()
+        remaining: List[int] = list()
+
+        self.on_enter(keys, workers, spec)
+
+        with Pool(workers) as pool:
+            mapper = pool.imap_unordered(func, keys)
+
+            for key, error in mapper:
+                if error is None:
+                    converted.append(key)
+                    self.on_success(key)
+                else:
+                    remaining.append(key)
+                    self.on_failure(key, error)
+
+        self.on_exit(converted, remaining)