-
Notifications
You must be signed in to change notification settings - Fork 1
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
[dashboard] Added Support for ZIM-discovery
On start, a new script is called to list ZIM files in expected location and updates the Packages YAML Then the gen-home script is called to generate home from (updated) Packages YAML. A healthcheck is also added to inform outside that the home is ready so other services depending on updates Packages.yaml can start
- Loading branch information
Showing
4 changed files
with
212 additions
and
23 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,14 @@ | ||
| #!/bin/sh | ||
| set -e | ||
|
|
||
| VENV=/usr/local/bin/gen-home_env | ||
|
|
||
| # refresh ZIM packages collection (maybe) | ||
| $VENV/bin/python3 $VENV/lib/refresh-zims.cpython-311.pyc | ||
|
|
||
| # generate homepage from collection | ||
| $VENV/bin/python3 $VENV/lib/gen-home.cpython-311.pyc | ||
|
|
||
| touch /tmp/.ready | ||
|
|
||
| exec "$@" |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,150 @@ | ||
| #!/usr/bin/env python3 | ||
|
|
||
| """ refresh-zims: update YAML packages list using disk-discovered ZIM files | ||
| This is an optional feature, yet it is enabled by default. | ||
| Set `DONT_UPDATE_PACKAGES` environ to disable | ||
| Once enabled, this loops over ZIM files present on disk (at `ZIM_DIR` path) | ||
| and updates the Packages YAML (at `PACKAGES_PATH`) accordingly: | ||
| - if the package (using its ident) was in YAML: keep YAML entry | ||
| - if the ZIM was not in YAML: add entry using in-ZIM info | ||
| - entries in YAML for which ZIM is missing are disabled but kept. | ||
| Dependencies: | ||
| - PyYAML | ||
| - libzim | ||
| """ | ||
| import base64 | ||
| import os | ||
| import pathlib | ||
| import traceback | ||
| from typing import Any | ||
|
|
||
| import yaml | ||
| from libzim.reader import Archive | ||
|
|
||
| try: | ||
| from yaml import CDumper as Dumper | ||
| from yaml import CSafeLoader as SafeLoader | ||
| except ImportError: | ||
| # we don't NEED cython ext but it's faster so use it if avail. | ||
| from yaml import Dumper, SafeLoader | ||
|
|
||
| packages_path = ( | ||
| pathlib.Path(os.getenv("PACKAGES_PATH", "home.yaml")).expanduser().resolve() | ||
| ) | ||
| zims_dir = pathlib.Path(os.getenv("ZIM_DIR", "/src")) | ||
| dont_update = bool(os.getenv("DONT_UPDATE_PACKAGES", "")) | ||
|
|
||
| kiwix_reader_link_tpl = os.getenv("KIWIX_READER_LINK_TPL", "//kiwix.{fqdn}/{zim_name}") | ||
| kiwix_download_link_tpl = os.getenv( | ||
| "KIWIX_DOWNLOAD_LINK_TPL", "//kiwixdl.{fqdn}/{zim_filename}" | ||
| ) | ||
|
|
||
|
|
||
| def get_metadata(archive: Archive, name: str) -> str: | ||
| if name not in archive.metadata_keys: | ||
| return "" | ||
| return archive.get_metadata(name).decode("UTF-8") | ||
|
|
||
|
|
||
| def get_kiwix_url(template: str, fqdn: str, name: str, filename: str) -> str: | ||
| return ( | ||
| template.replace("{fqdn}", fqdn) | ||
| .replace("{zim_name}", name) | ||
| .replace("{zim_filename}", filename) | ||
| ) | ||
|
|
||
|
|
||
| def get_entry_for( | ||
| fpath: pathlib.Path, document_metadata: dict[str, str] | ||
| ) -> dict[str, Any]: | ||
| zim = Archive(fpath) | ||
| publisher = get_metadata(zim, "Publisher") | ||
| name = get_metadata(zim, "Name") | ||
| flavour = get_metadata(zim, "Flavour") | ||
| ident = f"{publisher}:{name}:{flavour}" | ||
| icon = None | ||
| if zim.has_illustration and 48 in zim.get_illustration_sizes(): | ||
| icon = base64.b64encode(bytes(zim.get_illustration_item(48).content)).decode( | ||
| "ASCII" | ||
| ) | ||
| return { | ||
| "kind": "zim", | ||
| "ident": ident, | ||
| "title": get_metadata(zim, "Title"), | ||
| "description": get_metadata(zim, "Description"), | ||
| "languages": get_metadata(zim, "Language").split(",") or ["eng"], | ||
| "tags": get_metadata(zim, "Tags").split(";"), | ||
| "url": get_kiwix_url( | ||
| template=kiwix_reader_link_tpl, | ||
| fqdn=document_metadata["fqdn"], | ||
| name=name, | ||
| filename=fpath.name, | ||
| ), | ||
| "download": { | ||
| "url": get_kiwix_url( | ||
| template=kiwix_download_link_tpl, | ||
| fqdn=document_metadata["fqdn"], | ||
| name=name, | ||
| filename=fpath.name, | ||
| ), | ||
| "size": fpath.stat().st_size, | ||
| }, | ||
| "icon": icon, | ||
| } | ||
|
|
||
|
|
||
| def refresh_zims( | ||
| packages_path: pathlib.Path, zims_dir: pathlib.Path, debug: bool | None = False | ||
| ): | ||
| print(f"refreshing ZIMs from {zims_dir=}") | ||
| try: | ||
| document = yaml.load(packages_path.read_text(), Loader=SafeLoader) | ||
| document["packages"] | ||
| document["metadata"] | ||
| document["metadata"]["fqdn"] | ||
| document["metadata"]["name"] | ||
| except Exception as exc: | ||
| print("[CRITICAL] unable to read home YAML document, skiping") | ||
| traceback.print_exception(exc) | ||
| return | ||
|
|
||
| # copy list of packages from YAML | ||
| conf_packages = { | ||
| package.get("ident", ""): package for package in document["packages"] | ||
| } | ||
| document["packages"] = [] | ||
|
|
||
| for zim_fpath in zims_dir.glob("*.zim"): | ||
| try: | ||
| package = get_entry_for(zim_fpath, document["metadata"]) | ||
| except Exception as exc: | ||
| print(f"Failed to read from {zim_fpath}, skiping ({exc})") | ||
| continue | ||
|
|
||
| # reuse package definition from YAML | ||
| if package["ident"] in conf_packages: | ||
| document["packages"].append(conf_packages[package["ident"]]) | ||
| continue | ||
|
|
||
| # use from-zim package definition from discovered ZIM | ||
| document["packages"].append(package) | ||
| try: | ||
| packages_path.write_text(yaml.dump(document, Dumper=Dumper)) | ||
| except Exception as exc: | ||
| print("[CRITICAL] unable to update(save) home YAML document, skiping") | ||
| traceback.print_exception(exc) | ||
| return | ||
|
|
||
|
|
||
| if __name__ == "__main__": | ||
| if not dont_update and zims_dir.exists() and zims_dir.is_dir(): | ||
| refresh_zims( | ||
| packages_path=packages_path, | ||
| zims_dir=zims_dir, | ||
| debug=bool(os.getenv("DEBUG", False)), | ||
| ) | ||
| else: | ||
| print(f"Not refreshing ZIMs for {dont_update=}, {zims_dir=}") |