From 30e536be088587d8cdbbd1876517e07dc120ce39 Mon Sep 17 00:00:00 2001 From: Wei-Cheng Pan Date: Sat, 16 Dec 2023 20:12:48 +0900 Subject: [PATCH] move files --- {driveutil/fix-hah => drive}/Makefile | 0 {driveutil/fix-hah => drive}/README.md | 0 .../faststart => drive}/app/__init__.py | 0 .../fix-jav/app => drive/app/hah}/__init__.py | 0 .../fix-hah/app => drive/app/hah}/__main__.py | 0 .../fix-hah/app => drive/app/hah}/_archive.py | 0 .../fix-hah/app => drive/app/hah}/_arg.py | 2 +- .../fix-hah/app => drive/app/hah}/_check.py | 0 .../app => drive/app/hah}/_download.py | 4 +- .../fix-hah/app => drive/app/hah}/_main.py | 0 .../fix-hah/app => drive/app/hah}/_upload.py | 0 drive/app/jav/__init__.py | 0 .../fix-jav/app => drive/app/jav}/__main__.py | 2 +- drive/app/jav/_dispatch.py | 174 ++++++++++++ drive/app/jav/_main.py | 134 ++++++++++ drive/app/jav/_sauce.py | 217 +++++++++++++++ drive/app/jav/_types.py | 13 + drive/app/lib/__init__.py | 0 drive/hah.sh | 3 + drive/legacy/faststart/app/__init__.py | 0 .../legacy}/faststart/app/__main__.py | 0 .../legacy}/faststart/app/cache.py | 0 .../legacy}/faststart/app/main.py | 0 .../legacy}/faststart/app/processor.py | 0 .../legacy}/migration/.dockerignore | 0 .../legacy}/migration/.gitignore | 0 .../legacy}/migration/Dockerfile | 0 .../legacy}/migration/app/common.py | 0 .../legacy}/migration/app/migrate.py | 0 .../legacy}/migration/app/progress.py | 0 .../legacy}/migration/app/speed.py | 0 .../legacy}/migration/docker-compose.yml | 0 {driveutil => drive/legacy}/shell/finddup.py | 0 {driveutil => drive/legacy}/shell/fix-jav | 0 {driveutil => drive/legacy}/shell/fix-media | 0 {driveutil => drive/legacy}/shell/fix-unicode | 0 {driveutil => drive/legacy}/shell/fixcache.py | 0 {driveutil/fix-hah => drive}/poetry.lock | 0 {driveutil/fix-hah => drive}/poetry.toml | 0 {driveutil/fix-hah => drive}/pyproject.toml | 0 driveutil/fix-hah/run.sh | 3 - driveutil/fix-jav/app/dispatch.py | 86 ------ driveutil/fix-jav/app/main.py | 126 --------- driveutil/fix-jav/app/sauce.py | 248 ------------------ 44 files changed, 546 insertions(+), 466 deletions(-) rename {driveutil/fix-hah => drive}/Makefile (100%) rename {driveutil/fix-hah => drive}/README.md (100%) rename {driveutil/faststart => drive}/app/__init__.py (100%) rename {driveutil/fix-jav/app => drive/app/hah}/__init__.py (100%) rename {driveutil/fix-hah/app => drive/app/hah}/__main__.py (100%) rename {driveutil/fix-hah/app => drive/app/hah}/_archive.py (100%) rename {driveutil/fix-hah/app => drive/app/hah}/_arg.py (93%) rename {driveutil/fix-hah/app => drive/app/hah}/_check.py (100%) rename {driveutil/fix-hah/app => drive/app/hah}/_download.py (93%) rename {driveutil/fix-hah/app => drive/app/hah}/_main.py (100%) rename {driveutil/fix-hah/app => drive/app/hah}/_upload.py (100%) create mode 100644 drive/app/jav/__init__.py rename {driveutil/fix-jav/app => drive/app/jav}/__main__.py (71%) create mode 100644 drive/app/jav/_dispatch.py create mode 100644 drive/app/jav/_main.py create mode 100644 drive/app/jav/_sauce.py create mode 100644 drive/app/jav/_types.py create mode 100644 drive/app/lib/__init__.py create mode 100755 drive/hah.sh create mode 100644 drive/legacy/faststart/app/__init__.py rename {driveutil => drive/legacy}/faststart/app/__main__.py (100%) rename {driveutil => drive/legacy}/faststart/app/cache.py (100%) rename {driveutil => drive/legacy}/faststart/app/main.py (100%) rename {driveutil => drive/legacy}/faststart/app/processor.py (100%) rename {driveutil => drive/legacy}/migration/.dockerignore (100%) rename {driveutil => drive/legacy}/migration/.gitignore (100%) rename {driveutil => drive/legacy}/migration/Dockerfile (100%) rename {driveutil => drive/legacy}/migration/app/common.py (100%) rename {driveutil => drive/legacy}/migration/app/migrate.py (100%) rename {driveutil => drive/legacy}/migration/app/progress.py (100%) rename {driveutil => drive/legacy}/migration/app/speed.py (100%) rename {driveutil => drive/legacy}/migration/docker-compose.yml (100%) rename {driveutil => drive/legacy}/shell/finddup.py (100%) rename {driveutil => drive/legacy}/shell/fix-jav (100%) rename {driveutil => drive/legacy}/shell/fix-media (100%) rename {driveutil => drive/legacy}/shell/fix-unicode (100%) rename {driveutil => drive/legacy}/shell/fixcache.py (100%) rename {driveutil/fix-hah => drive}/poetry.lock (100%) rename {driveutil/fix-hah => drive}/poetry.toml (100%) rename {driveutil/fix-hah => drive}/pyproject.toml (100%) delete mode 100755 driveutil/fix-hah/run.sh delete mode 100644 driveutil/fix-jav/app/dispatch.py delete mode 100644 driveutil/fix-jav/app/main.py delete mode 100644 driveutil/fix-jav/app/sauce.py diff --git a/driveutil/fix-hah/Makefile b/drive/Makefile similarity index 100% rename from driveutil/fix-hah/Makefile rename to drive/Makefile diff --git a/driveutil/fix-hah/README.md b/drive/README.md similarity index 100% rename from driveutil/fix-hah/README.md rename to drive/README.md diff --git a/driveutil/faststart/app/__init__.py b/drive/app/__init__.py similarity index 100% rename from driveutil/faststart/app/__init__.py rename to drive/app/__init__.py diff --git a/driveutil/fix-jav/app/__init__.py b/drive/app/hah/__init__.py similarity index 100% rename from driveutil/fix-jav/app/__init__.py rename to drive/app/hah/__init__.py diff --git a/driveutil/fix-hah/app/__main__.py b/drive/app/hah/__main__.py similarity index 100% rename from driveutil/fix-hah/app/__main__.py rename to drive/app/hah/__main__.py diff --git a/driveutil/fix-hah/app/_archive.py b/drive/app/hah/_archive.py similarity index 100% rename from driveutil/fix-hah/app/_archive.py rename to drive/app/hah/_archive.py diff --git a/driveutil/fix-hah/app/_arg.py b/drive/app/hah/_arg.py similarity index 93% rename from driveutil/fix-hah/app/_arg.py rename to drive/app/hah/_arg.py index 8a19eb1c..69509c95 100644 --- a/driveutil/fix-hah/app/_arg.py +++ b/drive/app/hah/_arg.py @@ -10,7 +10,7 @@ class KeywordArgument: def parse_args(args: list[str]) -> KeywordArgument: - parser = ArgumentParser("fix-hah") + parser = ArgumentParser("hah") parser.add_argument("--config", "-c", required=True, type=str) parser.add_argument("path", nargs="+", type=str) diff --git a/driveutil/fix-hah/app/_check.py b/drive/app/hah/_check.py similarity index 100% rename from driveutil/fix-hah/app/_check.py rename to drive/app/hah/_check.py diff --git a/driveutil/fix-hah/app/_download.py b/drive/app/hah/_download.py similarity index 93% rename from driveutil/fix-hah/app/_download.py rename to drive/app/hah/_download.py index dc4f4159..1445514e 100644 --- a/driveutil/fix-hah/app/_download.py +++ b/drive/app/hah/_download.py @@ -38,7 +38,9 @@ async def _download_directory( await queue.push(_download_unknown(queue, drive, child, new_direcotry, pool)) -async def _download_file(queue: AioQueue[None], drive: Drive, src: Node, dst: Path, pool: Executor) -> None: +async def _download_file( + queue: AioQueue[None], drive: Drive, src: Node, dst: Path, pool: Executor +) -> None: path = await download_file_to_local(drive, src, dst) print(f"touch {path}") diff --git a/driveutil/fix-hah/app/_main.py b/drive/app/hah/_main.py similarity index 100% rename from driveutil/fix-hah/app/_main.py rename to drive/app/hah/_main.py diff --git a/driveutil/fix-hah/app/_upload.py b/drive/app/hah/_upload.py similarity index 100% rename from driveutil/fix-hah/app/_upload.py rename to drive/app/hah/_upload.py diff --git a/drive/app/jav/__init__.py b/drive/app/jav/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/driveutil/fix-jav/app/__main__.py b/drive/app/jav/__main__.py similarity index 71% rename from driveutil/fix-jav/app/__main__.py rename to drive/app/jav/__main__.py index 416ab98a..48e18a27 100644 --- a/driveutil/fix-jav/app/__main__.py +++ b/drive/app/jav/__main__.py @@ -1,7 +1,7 @@ import asyncio import sys -from .main import main +from ._main import main sys.exit(asyncio.run(main())) diff --git a/drive/app/jav/_dispatch.py b/drive/app/jav/_dispatch.py new file mode 100644 index 00000000..bbc287bd --- /dev/null +++ b/drive/app/jav/_dispatch.py @@ -0,0 +1,174 @@ +import re + +from ._types import SauceData, JavData + + +def match_jav_1(name: str) -> JavData | None: + m = re.search(r"(\w{2,6})[-_](\d{2,4}\w?)", name) + if not m: + return None + name = f"{m.group(1)}-{m.group(2)}" + name = name.upper() + return JavData( + name=name, + sauce_list=[ + SauceData(name="javbus", query=name), + SauceData(name="javlibrary", query=name), + ], + ) + + +def match_jav_2(name: str) -> JavData | None: + m = re.search(r"(\d{3,4})(\w{3,6})[-_](\d{3,4}\w?)", name) + if not m: + return None + name = f"{m.group(1)}{m.group(2)}-{m.group(3)}" + name = name.upper() + return JavData( + name=name, + sauce_list=[ + SauceData(name="javbus", query=name), + SauceData(name="javlibrary", query=name), + ], + ) + + +def match_fc2(name: str) -> JavData | None: + m = re.search(r"fc2[-_]ppv[-_](\d+)", name, re.I) + if not m: + return None + name = f"FC2-PPV-{m.group(1)}" + return JavData( + name=name, + sauce_list=[ + SauceData(name="javbee", query=name), + SauceData(name="javtorrent", query=name), + ], + ) + + +def match_heydouga(name: str) -> JavData | None: + m = re.search(r"hey(douga)?[-_ ]?(\d+)[-_](\d+)", name, re.I) + if not m: + return None + a = m.group(2) + b = m.group(3) + name = f"HEYDOUGA-{a}-{b}" + return JavData( + name=name, + sauce_list=[ + SauceData(name="heydouga", query=f"{a}/{b}"), + ], + ) + + +def match_caribpr(name: str) -> JavData | None: + m = re.search(r"(\d{6})[-_](\d{3})-CARIBPR", name, re.I) + if not m: + return None + a = m.group(1) + b = m.group(2) + name = f"CARIBPR {a}-{b}" + return JavData( + name=name, + sauce_list=[SauceData(name="caribpr", query=f"{a}-{b}")], + ) + + +def match_carib(name: str) -> JavData | None: + m = re.search(r"(\d{6})[-_](\d{3})-CARIB", name, re.I) + if not m: + return None + a = m.group(1) + b = m.group(2) + name = f"CARIB {a}-{b}" + return JavData( + name=name, + sauce_list=[SauceData(name="carib", query=f"{a}-{b}")], + ) + + +def match_1pondo(name: str) -> JavData | None: + m = re.search(r"(\d{6})[-_](\d{3})-1PON", name, re.I) + if not m: + return None + a = m.group(1) + b = m.group(2) + name = f"1PONDO {a}-{b}" + return JavData( + name=name, + sauce_list=[SauceData(name="1pondo", query=f"{a}-{b}")], + ) + + +def match_unknown(name: str) -> JavData | None: + m = re.search(r"(\d{6})[-_](\d{3})", name, re.I) + if not m: + return None + a = m.group(1) + b = m.group(2) + name = f"{a}-{b}" + return JavData( + name=name, + sauce_list=[ + SauceData(name="carib", query=f"{a}-{b}"), + SauceData(name="caribpr", query=f"{a}-{b}"), + SauceData(name="1pondo", query=f"{a}-{b}"), + ], + ) + + +def match_mesubuta(name: str) -> JavData | None: + m = re.search(r"(\d{6})[-_](\d{3})[-_](\d{2})", name, re.I) + if not m: + return None + name = f"{m.group(1)}_{m.group(2)}_{m.group(3)}" + return JavData( + name=name, + sauce_list=[], + ) + + +def match_10musume(name: str) -> JavData | None: + m = re.search(r"(\d{6})[-_](\d{2})-10MU", name, re.I) + if not m: + return None + a = m.group(1) + b = m.group(2) + name = f"10MU {a}_{b}" + return JavData( + name=name, + sauce_list=[ + SauceData(name="10musume", query=f"{a}_{b}"), + ], + ) + + +EXCLUDE_LIST = [ + match_mesubuta, +] + + +INCLUDE_LIST = [ + match_10musume, + match_caribpr, + match_carib, + match_1pondo, + match_heydouga, + match_fc2, + match_jav_1, + match_jav_2, + match_unknown, +] + + +def get_jav_query(name: str) -> JavData | None: + for ex in EXCLUDE_LIST: + rv = ex(name) + if rv: + return None + for in_ in INCLUDE_LIST: + rv = in_(name) + if rv: + return rv + return None diff --git a/drive/app/jav/_main.py b/drive/app/jav/_main.py new file mode 100644 index 00000000..13230de8 --- /dev/null +++ b/drive/app/jav/_main.py @@ -0,0 +1,134 @@ +from argparse import ArgumentParser, Namespace +from collections.abc import Callable, Awaitable +from pathlib import Path +from typing import Any +import asyncio +import sys + +import yaml +from aiohttp import ClientSession +from wcpan.drive.core.types import Drive, Node +from wcpan.drive.cli.lib import create_drive_from_config + +from ._sauce import fetch_jav_data +from ._dispatch import get_jav_query + + +async def main(args: list[str] | None = None) -> int: + parser = ArgumentParser("jav") + + parser.add_argument("--config", "-c", required=True, type=str) + + command = parser.add_subparsers() + + g_parser = command.add_parser("generate", aliases=["g"]) + g_parser.add_argument("path", required=True, type=str) + g_parser.set_defaults(action=_generate) + + a_parser = command.add_parser("apply", aliases=["a"]) + a_parser.set_defaults(action=_apply) + + f_parser = command.add_parser("filter", aliases=["f"]) + f_parser.set_defaults(action=_filter) + + kwargs = parser.parse_args(args) + + config_path = Path(kwargs.config).expanduser().resolve() + action: Callable[[Drive, Namespace], Awaitable[int]] | None = kwargs.action + if not action: + return 1 + + async with create_drive_from_config(config_path) as drive: + return await action(drive, kwargs) + + +async def _generate(drive: Drive, kwargs: Namespace) -> int: + root_path = Path(kwargs.path) + + async with ClientSession() as session: + root_node = await drive.get_node_by_path(root_path) + children = await drive.get_children(root_node) + async for node in process_node_list(session, children): + yaml.safe_dump( + [node], + sys.stdout, + encoding="utf-8", + allow_unicode=True, + default_flow_style=False, + ) + await asyncio.sleep(1) + return 0 + + +async def _apply(drive: Drive, kwargs: Namespace) -> int: + manifest = yaml.safe_load(sys.stdin) + for row in manifest: + id_ = row["id"] + title_dict = row["title"] + + for value in title_dict.values(): + if not value: + continue + + node = await drive.get_node_by_id(id_) + print(f"rename {node.name} -> {value}") + await rename(drive, node, value) + break + return 0 + + +async def _filter(drive: Drive, kwargs: Namespace) -> int: + manifest = yaml.safe_load(sys.stdin) + + not_all_null = (m for m in manifest if any(m["title"].values())) + + def all_same(m: dict[str, Any]): + values = set(v for v in m["title"].values() if v) + values.add(m["name"]) + return len(values) == 1 + + not_all_same = (m for m in not_all_null if not all_same(m)) + + for node in not_all_same: + yaml.safe_dump( + [node], + sys.stdout, + encoding="utf-8", + allow_unicode=True, + default_flow_style=False, + ) + + return 0 + + +async def process_node_list(session: ClientSession, node_list: list[Node]): + for node in node_list: + if node.is_trashed: + continue + jav_query = get_jav_query(node.name) + if not jav_query: + continue + title = await fetch_jav_data(session, jav_query) + + yield { + "id": node.id, + "name": node.name, + "jav_id": jav_query, + "title": title, + } + + +async def rename(drive: Drive, node: Node, new_name: str) -> None: + if node.is_directory: + if new_name == node.name: + print("skipped") + return + await drive.move(node, new_parent=None, new_name=new_name) + return + + if not node.parent_id: + return + + root_node = await drive.get_node_by_id(node.parent_id) + parent = await drive.create_directory(new_name, root_node) + await drive.move(node, new_parent=parent, new_name=None) diff --git a/drive/app/jav/_sauce.py b/drive/app/jav/_sauce.py new file mode 100644 index 00000000..9620b3a8 --- /dev/null +++ b/drive/app/jav/_sauce.py @@ -0,0 +1,217 @@ +import re +from asyncio import as_completed +from collections.abc import Awaitable + +from aiohttp import ClientSession +from bs4 import BeautifulSoup + +from ._types import JavData + + +async def fetch_jav_data_from_javbus(session: ClientSession, jav_id: str, query: str): + async with session.get(f"https://www.javbus.com/ja/{query}") as response: + if response.status != 200: + return None + + html = await response.text(errors="ignore") + soup = BeautifulSoup(html, "html.parser") + title = soup.select_one(".container > h3") + if not title: + return None + return normalize_title(title.text) + + +async def fetch_jav_data_from_javlibrary( + session: ClientSession, jav_id: str, query: str +): + async with session.get( + f"http://www.javlibrary.com/ja/vl_searchbyid.php", + params={ + "keyword": query, + }, + ) as response: + if response.status != 200: + return None + + html = await response.text(errors="ignore") + soup = BeautifulSoup(html, "html.parser") + title = soup.select_one("#video_title .post-title") + if title: + return normalize_title(title.text) + + videos = soup.select(".videos .video") + for div in videos: + id_ = div.select_one(".id") + if not id_: + continue + if id_.text != query: + continue + a = div.select_one("a") + if a: + title = a.get("title") + if not title: + continue + if title.find("（ブルーレイディスク）") >= 0: + continue + return normalize_title(title) + + return None + + +async def fetch_jav_data_from_javbee(session: ClientSession, jav_id: str, query: str): + async with session.get( + "https://javbee.org/search", + params={ + "keyword": query, + }, + ) as response: + if response.status != 200: + return None + + html = await response.text(errors="ignore") + soup = BeautifulSoup(html, "html.parser") + title = soup.select_one(".title > a") + if not title: + return None + return normalize_title(title.text) + + +async def fetch_jav_data_from_heydouga(session: ClientSession, jav_id: str, query: str): + async with session.get( + f"https://www.heydouga.com/moviepages/{query}/index.html", + ) as response: + if response.status != 200: + return None + + html = await response.text(errors="ignore") + soup = BeautifulSoup(html, "html.parser") + title = soup.select_one("#title-bg > h1") + if not title: + return None + for span in title.find_all("span"): + span.decompose() + return f"{jav_id} {normalize_title(title.text)}" + + +async def fetch_jav_data_from_carib(session: ClientSession, jav_id: str, query: str): + async with session.get( + f"https://www.caribbeancom.com/moviepages/{query}/index.html", + ) as response: + if response.status != 200: + return None + + html = await response.text(errors="ignore") + soup = BeautifulSoup(html, "html.parser") + + title = soup.select_one("h1[itemprop=name]") + if not title: + return None + title = normalize_title(title.text) + + actor = soup.select_one(".movie-spec a[itemprop=actor] > span[itemprop=name]") + if not actor: + return f"{jav_id} {title}" + + return f"{jav_id} {title} {normalize_title(actor.text)}" + + +async def fetch_jav_data_from_caribpr(session: ClientSession, jav_id: str, query: str): + async with session.get( + f"https://www.caribbeancompr.com/moviepages/{query}/index.html", + ) as response: + if response.status != 200: + return None + + html = await response.text(errors="ignore") + soup = BeautifulSoup(html, "html.parser") + + title = soup.select_one(".movie-info .heading") + if not title: + return None + title = normalize_title(title.text) + + actor = soup.select_one(".movie-spec .spec-content > .spec-item") + if not actor: + return f"{jav_id} {title}" + + return f"{jav_id} {title} {normalize_title(actor.text)}" + + +async def fetch_jav_data_from_1pondo(session: ClientSession, jav_id: str, query: str): + m = re.match(r"\d{6}_\d{3}", jav_id) + if not m: + return None + + async with session.get( + f"https://www.1pondo.tv/dyn/phpauto/movie_details/movie_id/{query}.json", + ) as response: + if response.status != 200: + return None + + data = await response.json() + title = normalize_title(data["Title"]) + actor = normalize_title(data["Actor"]) + + return f"{jav_id} {title} {actor}" + + +async def fetch_jav_data_from_10musume(session: ClientSession, jav_id: str, query: str): + async with session.get( + f"https://www.10musume.com/dyn/phpauto/movie_details/movie_id/{jav_id}.json", + ) as response: + if response.status != 200: + return None + + data = await response.json() + title = normalize_title(data["Title"]) + actor = normalize_title(data["Actor"]) + + return f"10MU {jav_id} {title} {actor}" + + +# SOURCE_LIST = [ +# fetch_jav_data_from_javbus, +# fetch_jav_data_from_javlibrary, +# fetch_jav_data_from_javbee, +# fetch_jav_data_from_heydouga, +# fetch_jav_data_from_carib, +# fetch_jav_data_from_caribpr, +# fetch_jav_data_from_1pondo, +# fetch_jav_data_from_10musume, +# ] + +SAUCE_DICT = { + "javbus": fetch_jav_data_from_javbus, + "javlibrary": fetch_jav_data_from_javlibrary, + "javbee": fetch_jav_data_from_javbee, + "javtorrent": fetch_jav_data_from_javbee, + "heydouga": fetch_jav_data_from_heydouga, + "carib": fetch_jav_data_from_carib, + "caribpr": fetch_jav_data_from_caribpr, + "1pondo": fetch_jav_data_from_1pondo, + "10musume": fetch_jav_data_from_10musume, +} + + +async def fetch_jav_data(session: ClientSession, jav_query: JavData): + queries = ( + await _ + for _ in as_completed( + as_kv(_.name, SAUCE_DICT[_.name](session, jav_query.name, _.query)) + for _ in jav_query.sauce_list + ) + ) + rv = {k: v async for k, v in queries if v} + return rv + + +async def as_kv[K, V](key: K, value: Awaitable[V]) -> tuple[K, V]: + return key, await value + + +def normalize_title(title: str) -> str: + title = title.strip() + title = title.replace("/", "／") + title = title.replace("\n", "") + title = title.replace("\r", "") + return title diff --git a/drive/app/jav/_types.py b/drive/app/jav/_types.py new file mode 100644 index 00000000..9783cbef --- /dev/null +++ b/drive/app/jav/_types.py @@ -0,0 +1,13 @@ +from dataclasses import dataclass + + +@dataclass(frozen=True, kw_only=True) +class SauceData: + name: str + query: str + + +@dataclass(frozen=True, kw_only=True) +class JavData: + name: str + sauce_list: list[SauceData] diff --git a/drive/app/lib/__init__.py b/drive/app/lib/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/drive/hah.sh b/drive/hah.sh new file mode 100755 index 00000000..5873a70b --- /dev/null +++ b/drive/hah.sh @@ -0,0 +1,3 @@ +#! /bin/sh + +poetry run -- python3 -m app.hah -c "~/.config/wcpan.drive/cli.yaml" "$@" diff --git a/drive/legacy/faststart/app/__init__.py b/drive/legacy/faststart/app/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/driveutil/faststart/app/__main__.py b/drive/legacy/faststart/app/__main__.py similarity index 100% rename from driveutil/faststart/app/__main__.py rename to drive/legacy/faststart/app/__main__.py diff --git a/driveutil/faststart/app/cache.py b/drive/legacy/faststart/app/cache.py similarity index 100% rename from driveutil/faststart/app/cache.py rename to drive/legacy/faststart/app/cache.py diff --git a/driveutil/faststart/app/main.py b/drive/legacy/faststart/app/main.py similarity index 100% rename from driveutil/faststart/app/main.py rename to drive/legacy/faststart/app/main.py diff --git a/driveutil/faststart/app/processor.py b/drive/legacy/faststart/app/processor.py similarity index 100% rename from driveutil/faststart/app/processor.py rename to drive/legacy/faststart/app/processor.py diff --git a/driveutil/migration/.dockerignore b/drive/legacy/migration/.dockerignore similarity index 100% rename from driveutil/migration/.dockerignore rename to drive/legacy/migration/.dockerignore diff --git a/driveutil/migration/.gitignore b/drive/legacy/migration/.gitignore similarity index 100% rename from driveutil/migration/.gitignore rename to drive/legacy/migration/.gitignore diff --git a/driveutil/migration/Dockerfile b/drive/legacy/migration/Dockerfile similarity index 100% rename from driveutil/migration/Dockerfile rename to drive/legacy/migration/Dockerfile diff --git a/driveutil/migration/app/common.py b/drive/legacy/migration/app/common.py similarity index 100% rename from driveutil/migration/app/common.py rename to drive/legacy/migration/app/common.py diff --git a/driveutil/migration/app/migrate.py b/drive/legacy/migration/app/migrate.py similarity index 100% rename from driveutil/migration/app/migrate.py rename to drive/legacy/migration/app/migrate.py diff --git a/driveutil/migration/app/progress.py b/drive/legacy/migration/app/progress.py similarity index 100% rename from driveutil/migration/app/progress.py rename to drive/legacy/migration/app/progress.py diff --git a/driveutil/migration/app/speed.py b/drive/legacy/migration/app/speed.py similarity index 100% rename from driveutil/migration/app/speed.py rename to drive/legacy/migration/app/speed.py diff --git a/driveutil/migration/docker-compose.yml b/drive/legacy/migration/docker-compose.yml similarity index 100% rename from driveutil/migration/docker-compose.yml rename to drive/legacy/migration/docker-compose.yml diff --git a/driveutil/shell/finddup.py b/drive/legacy/shell/finddup.py similarity index 100% rename from driveutil/shell/finddup.py rename to drive/legacy/shell/finddup.py diff --git a/driveutil/shell/fix-jav b/drive/legacy/shell/fix-jav similarity index 100% rename from driveutil/shell/fix-jav rename to drive/legacy/shell/fix-jav diff --git a/driveutil/shell/fix-media b/drive/legacy/shell/fix-media similarity index 100% rename from driveutil/shell/fix-media rename to drive/legacy/shell/fix-media diff --git a/driveutil/shell/fix-unicode b/drive/legacy/shell/fix-unicode similarity index 100% rename from driveutil/shell/fix-unicode rename to drive/legacy/shell/fix-unicode diff --git a/driveutil/shell/fixcache.py b/drive/legacy/shell/fixcache.py similarity index 100% rename from driveutil/shell/fixcache.py rename to drive/legacy/shell/fixcache.py diff --git a/driveutil/fix-hah/poetry.lock b/drive/poetry.lock similarity index 100% rename from driveutil/fix-hah/poetry.lock rename to drive/poetry.lock diff --git a/driveutil/fix-hah/poetry.toml b/drive/poetry.toml similarity index 100% rename from driveutil/fix-hah/poetry.toml rename to drive/poetry.toml diff --git a/driveutil/fix-hah/pyproject.toml b/drive/pyproject.toml similarity index 100% rename from driveutil/fix-hah/pyproject.toml rename to drive/pyproject.toml diff --git a/driveutil/fix-hah/run.sh b/driveutil/fix-hah/run.sh deleted file mode 100755 index 870877b2..00000000 --- a/driveutil/fix-hah/run.sh +++ /dev/null @@ -1,3 +0,0 @@ -#! /bin/sh - -poetry run -- python3 -m app -c "~/.config/wcpan.drive/cli.yaml" "$@" diff --git a/driveutil/fix-jav/app/dispatch.py b/driveutil/fix-jav/app/dispatch.py deleted file mode 100644 index c8bda600..00000000 --- a/driveutil/fix-jav/app/dispatch.py +++ /dev/null @@ -1,86 +0,0 @@ -import re - - -def include_jav(name: str): - m = re.search(r'(\w{2,6})[-_](\d{2,4}\w?)', name) - if not m: - return '' - rv = f'{m.group(1)}-{m.group(2)}' - rv = rv.upper() - return rv - - -def include_jav2(name: str): - m = re.search(r'(\d{3,4})(\w{3,6})[-_](\d{3,4}\w?)', name) - if not m: - return '' - rv = f'{m.group(1)}{m.group(2)}-{m.group(3)}' - rv = rv.upper() - return rv - - -def include_fc2(name: str): - m = re.search(r'fc2[-_]ppv[-_](\d+)', name, re.I) - if not m: - return '' - rv = f'FC2-PPV-{m.group(1)}' - return rv - - -def include_heydouga(name: str): - m = re.search(r'hey(douga)?[-_ ]?(\d+)[-_](\d+)', name, re.I) - if not m: - return '' - rv = f'HEYDOUGA-{m.group(2)}-{m.group(3)}' - return rv - - -def include_carib(name: str): - m = re.search(r'(\d{6})[-_](\d{3})', name, re.I) - if not m: - return '' - rv = f'{m.group(1)}_{m.group(2)}' - return rv - - -def include_mesubuta(name: str): - m = re.search(r'(\d{6})[-_](\d{3})[-_](\d{2})', name, re.I) - if not m: - return '' - rv = f'{m.group(1)}_{m.group(2)}_{m.group(3)}' - return rv - - -def include_10musume(name: str): - m = re.search(r'(\d{6})[-_](\d{2})-10MU', name, re.I) - if not m: - return '' - rv = f'{m.group(1)}_{m.group(2)}' - return rv - - -EXCLUDE_LIST = [ - include_mesubuta, -] - - -INCLUDE_LIST = [ - include_10musume, - include_carib, - include_heydouga, - include_fc2, - include_jav2, - include_jav, -] - - -def get_jav_id(name: str) -> str: - for ex in EXCLUDE_LIST: - rv = ex(name) - if rv: - return '' - for in_ in INCLUDE_LIST: - rv = in_(name) - if rv: - return rv - return '' diff --git a/driveutil/fix-jav/app/main.py b/driveutil/fix-jav/app/main.py deleted file mode 100644 index d17ed03d..00000000 --- a/driveutil/fix-jav/app/main.py +++ /dev/null @@ -1,126 +0,0 @@ -import asyncio -import sys - -import yaml -from aiohttp import ClientSession -from wcpan.drive.core.drive import DriveFactory, Drive, Node - -from .sauce import fetch_jav_data -from .dispatch import get_jav_id - - -async def main(args: list[str] = None): - if args is None: - args = sys.argv - - args = args[1:] - mode = args[0] - args = args[1:] - if mode == 'generate' or mode == 'g': - await generate_manifest(args) - elif mode == 'apply' or mode == 'a': - await apply_manifest(args) - elif mode == 'filter' or mode == 'f': - await filter_manifest(args) - else: - return 1 - - return 0 - - -async def generate_manifest(args: list[str]): - root_path = args[0] - - factory = DriveFactory() - factory.load_config() - - async with factory() as drive, \ - ClientSession() as session: - root_node = await drive.get_node_by_path(root_path) - children = await drive.get_children(root_node) - async for node in process_node_list(session, children): - yaml.safe_dump( - [node], - sys.stdout, - encoding='utf-8', - allow_unicode=True, - default_flow_style=False, - ) - await asyncio.sleep(1) - - -async def apply_manifest(args: list[str]): - factory = DriveFactory() - factory.load_config() - - manifest = yaml.safe_load(sys.stdin) - async with factory() as drive: - for row in manifest: - id_ = row['id'] - title_dict = row['title'] - - for value in title_dict.values(): - if not value: - continue - - node = await drive.get_node_by_id(id_) - print(f'rename {node.name} -> {value}') - await rename(drive, node, value) - break - - -async def filter_manifest(args: list[str]): - manifest = yaml.safe_load(sys.stdin) - - not_all_null = (m for m in manifest if any(m['title'].values())) - - def all_same(m): - values = set(v for v in m['title'].values() if v) - values.add(m['name']) - return len(values) == 1 - not_all_same = (m for m in not_all_null if not all_same(m)) - - for node in not_all_same: - yaml.safe_dump( - [node], - sys.stdout, - encoding='utf-8', - allow_unicode=True, - default_flow_style=False, - ) - - -async def process_node_list(session: ClientSession, node_list: list[Node]): - for node in node_list: - if node.trashed: - continue - jav_id = get_jav_id(node.name) - if not jav_id: - continue - title = await fetch_jav_data(session, jav_id) - - name_set = set() - name_set.add(node.name) - for value in title.values(): - name_set.add(value) - if len(name_set) <= 1: - continue - - yield { - 'id': node.id_, - 'name': node.name, - 'jav_id': jav_id, - 'title': title, - } - - -async def rename(drive: Drive, node: Node, new_name: str): - if node.is_folder: - if new_name == node.name: - print('skipped') - return - await drive.rename_node(node, new_parent=None, new_name=new_name) - return - root_node = await drive.get_node_by_id(node.parent_id) - parent = await drive.create_folder(root_node, new_name) - await drive.rename_node(node, new_parent=parent, new_name=None) diff --git a/driveutil/fix-jav/app/sauce.py b/driveutil/fix-jav/app/sauce.py deleted file mode 100644 index 50f5f814..00000000 --- a/driveutil/fix-jav/app/sauce.py +++ /dev/null @@ -1,248 +0,0 @@ -import asyncio -import functools -import re - -from aiohttp import ClientSession -from bs4 import BeautifulSoup - - -def named_fetch(name: str): - def decorator(fn): - @functools.wraps(fn) - async def wrapper(*args, **kwargs): - rv = await fn(*args, **kwargs) - return name, rv - return wrapper - return decorator - - -@named_fetch('javbus') -async def fetch_jav_data_from_javbus(session: ClientSession, jav_id: str): - if jav_id.startswith('FC2-PPV'): - return None - - async with session.get(f'https://www.javbus.com/ja/{jav_id}') as response: - if response.status != 200: - return None - - html = await response.text(errors='ignore') - soup = BeautifulSoup(html, 'html.parser') - title = soup.select_one('.container > h3') - if not title: - return None - return normalize_title(title.text) - - -@named_fetch('javlib') -async def fetch_jav_data_from_javlibrary(session: ClientSession, jav_id: str): - if jav_id.startswith('FC2-PPV'): - return None - - async with session.get( - f'http://www.javlibrary.com/ja/vl_searchbyid.php', - params={ - 'keyword': jav_id, - }, - ) as response: - if response.status != 200: - return None - - html = await response.text(errors='ignore') - soup = BeautifulSoup(html, 'html.parser') - title = soup.select_one('#video_title .post-title') - if title: - return normalize_title(title.text) - - videos = soup.select('.videos .video') - for div in videos: - id_ = div.select_one('.id') - if id_.text != jav_id: - continue - a = div.select_one('a') - if a: - title = a.get('title') - if title.find('（ブルーレイディスク）') >= 0: - continue - return normalize_title(title) - - return None - - -@named_fetch('javbee') -async def fetch_jav_data_from_javbee(session: ClientSession, jav_id: str): - if not jav_id.startswith('FC2-PPV'): - return None - - async with session.get('https://javbee.org/search', params={ - 'keyword': jav_id, - }) as response: - if response.status != 200: - return None - - html = await response.text(errors='ignore') - soup = BeautifulSoup(html, 'html.parser') - title = soup.select_one('.title > a') - if not title: - return None - return normalize_title(title.text) - - -@named_fetch('heydouga') -async def fetch_jav_data_from_heydouga(session: ClientSession, jav_id: str): - m = re.match(r'HEYDOUGA-(\d+)-(\d+)', jav_id) - if not m: - return None - - p1 = m.group(1) - p2 = m.group(2) - async with session.get( - f'https://www.heydouga.com/moviepages/{p1}/{p2}/index.html', - ) as response: - if response.status != 200: - return None - - html = await response.text(errors='ignore') - soup = BeautifulSoup(html, 'html.parser') - title = soup.select_one('#title-bg > h1') - if not title: - return None - for span in title.find_all('span'): - span.decompose() - return f'{jav_id} ' + normalize_title(title.text) - - -@named_fetch('carib') -async def fetch_jav_data_from_carib(session: ClientSession, jav_id: str): - m = re.match(r'(\d{6})_(\d{3})', jav_id) - if not m: - return None - - jav_id = f'{m.group(1)}-{m.group(2)}' - async with session.get( - f'https://www.caribbeancom.com/moviepages/{jav_id}/index.html', - ) as response: - if response.status != 200: - return None - - html = await response.text(errors='ignore') - soup = BeautifulSoup(html, 'html.parser') - - title = soup.select_one('h1[itemprop=name]') - if not title: - return None - title = normalize_title(title.text) - - actor = soup.select_one('.movie-spec a[itemprop=actor] > span[itemprop=name]') - if not actor: - actor = '' - else: - actor = normalize_title(actor.text) - - return f'CARIB {jav_id} {title} {actor}' - - -@named_fetch('caribpr') -async def fetch_jav_data_from_caribpr(session: ClientSession, jav_id: str): - m = re.match(r'\d{6}_\d{3}', jav_id) - if not m: - return None - - async with session.get( - f'https://www.caribbeancompr.com/moviepages/{jav_id}/index.html', - ) as response: - if response.status != 200: - return None - - html = await response.text(errors='ignore') - soup = BeautifulSoup(html, 'html.parser') - - title = soup.select_one('.movie-info .heading') - if not title: - return None - title = normalize_title(title.text) - - actor = soup.select_one('.movie-spec .spec-content > .spec-item') - if not actor: - actor = '' - else: - actor = normalize_title(actor.text) - - return f'CARIBPR {jav_id} {title} {actor}' - - -@named_fetch('1pondo') -async def fetch_jav_data_from_1pondo(session: ClientSession, jav_id: str): - m = re.match(r'\d{6}_\d{3}', jav_id) - if not m: - return None - - async with session.get( - f'https://www.1pondo.tv/dyn/phpauto/movie_details/movie_id/{jav_id}.json', - ) as response: - if response.status != 200: - return None - - data = await response.json() - title = normalize_title(data['Title']) - actor = normalize_title(data['Actor']) - - return f'1PONDO {jav_id} {title} {actor}' - - -@named_fetch('10musume') -async def fetch_jav_data_from_10musume(session: ClientSession, jav_id: str): - async with session.get( - f'https://www.10musume.com/dyn/phpauto/movie_details/movie_id/{jav_id}.json', - ) as response: - if response.status != 200: - return None - - data = await response.json() - title = normalize_title(data['Title']) - actor = normalize_title(data['Actor']) - - return f'10MU {jav_id} {title} {actor}' - - -SOURCE_LIST = [ - fetch_jav_data_from_javbus, - fetch_jav_data_from_javlibrary, - fetch_jav_data_from_javbee, - fetch_jav_data_from_heydouga, - fetch_jav_data_from_carib, - fetch_jav_data_from_caribpr, - fetch_jav_data_from_1pondo, - fetch_jav_data_from_10musume, -] - - -async def fetch_jav_data(session: ClientSession, jav_id: str): - tasks = [source(session, jav_id) for source in SOURCE_LIST] - tmp = await asyncio.gather(*tasks) - rv = {name: value for name, value in tmp} - return rv - - -def normalize_title(title: str) -> str: - title = title.strip() - title = title.replace('/', '／') - title = title.replace('\n', '') - title = title.replace('\r', '') - return title - - -async def fetch(jav_id: str): - async with ClientSession() as session: - rv = await fetch_jav_data_from_javlibrary(session, jav_id) - return rv - - -async def main(): - import sys - args = sys.argv - rv = await fetch(args[1]) - print(rv) - - -if __name__ == '__main__': - asyncio.run(main()) \ No newline at end of file