From e175aa73b14b6c16355089bdeb7df1216456e946 Mon Sep 17 00:00:00 2001 From: liblaf <30631553+liblaf@users.noreply.github.com> Date: Sun, 9 Jun 2024 20:43:57 +0800 Subject: [PATCH] feat: add CI workflow for automated builds and deployments - Added GitHub Actions workflow for CI - Configured build and deploy jobs - Uploaded artifacts for rule sets - Improved automation for project maintenance and deployment --- .cspell.json | 14 +++++ .envrc | 2 + .github/workflows/ci.yaml | 49 ++++++++++++++++++ .gitignore | 12 +++++ .vscode/settings.json | 7 +++ Makefile | 9 ++++ README.md | 4 ++ pyproject.toml | 31 +++++++++++ pyrightconfig.json | 3 ++ requirements-dev.lock | 30 +++++++++++ requirements.lock | 30 +++++++++++ scripts/build.py | 24 +++++++++ src/sbr/__init__.py | 0 src/sbr/__main__.py | 0 src/sbr/geosite.py | 71 ++++++++++++++++++++++++++ src/sbr/rule_set.py | 105 ++++++++++++++++++++++++++++++++++++++ 16 files changed, 391 insertions(+) create mode 100644 .cspell.json create mode 100644 .envrc create mode 100644 .github/workflows/ci.yaml create mode 100644 .gitignore create mode 100644 .vscode/settings.json create mode 100644 Makefile create mode 100644 README.md create mode 100644 pyproject.toml create mode 100644 pyrightconfig.json create mode 100644 requirements-dev.lock create mode 100644 requirements.lock create mode 100644 scripts/build.py create mode 100644 src/sbr/__init__.py create mode 100644 src/sbr/__main__.py create mode 100644 src/sbr/geosite.py create mode 100644 src/sbr/rule_set.py diff --git a/.cspell.json b/.cspell.json new file mode 100644 index 00000000..db852990 --- /dev/null +++ b/.cspell.json @@ -0,0 +1,14 @@ +{ + "words": [ + "dtemp", + "emby", + "liblaf", + "pycache", + "pydantic", + "pyproject", + "taplo", + "venv" + ], + "ignorePaths": ["**/*-lock.*", "**/*.lock*", "**/.cspell.json"], + "allowCompoundWords": true +} diff --git a/.envrc b/.envrc new file mode 100644 index 00000000..7b1d294e --- /dev/null +++ b/.envrc @@ -0,0 +1,2 @@ +# shellcheck disable=SC2148 +source_env_if_exists ./.venv/bin/activate diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml new file mode 100644 index 00000000..c6b8b53c --- /dev/null +++ b/.github/workflows/ci.yaml @@ -0,0 +1,49 @@ +name: CI + +on: + push: + schedule: + - cron: 0 0 * * * + +jobs: + build: + name: Build + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + - name: Setup Rye + uses: eifinger/setup-rye@v3 + - name: Install sing-box + uses: liblaf/repo/.github/actions/install@main + with: + brew: sing-box + - name: Install Dependencies + run: rye sync --no-lock + - name: Build Rule Sets + run: rye run python scripts/build.py + - name: Upload Artifacts + uses: actions/upload-artifact@v4 + with: + name: rule-sets + path: rule-sets + + deploy: + name: Deploy + permissions: + contents: write + needs: + - build + if: github.ref == 'refs/heads/main' + runs-on: ubuntu-latest + steps: + - name: Download Artifacts + uses: actions/download-artifact@v4 + with: + name: rule-sets + path: rule-sets + - name: Deploy to GitHub Branch + uses: peaceiris/actions-gh-pages@v4 + with: + publish_branch: rule-sets + publish_dir: rule-sets + force_orphan: true diff --git a/.gitignore b/.gitignore new file mode 100644 index 00000000..fa0ac030 --- /dev/null +++ b/.gitignore @@ -0,0 +1,12 @@ +# python generated files +__pycache__/ +*.py[oc] +build/ +dist/ +wheels/ +*.egg-info + +# venv +.venv + +rule-sets/ diff --git a/.vscode/settings.json b/.vscode/settings.json new file mode 100644 index 00000000..a59cab3e --- /dev/null +++ b/.vscode/settings.json @@ -0,0 +1,7 @@ +{ + "files.exclude": { + "**/__pycache__": true, + "**/.venv": true + }, + "python.analysis.diagnosticMode": "workspace" +} diff --git a/Makefile b/Makefile new file mode 100644 index 00000000..f4090076 --- /dev/null +++ b/Makefile @@ -0,0 +1,9 @@ +default: fmt + +fmt: fmt-toml + +fmt-toml: fmt-toml/pyproject.toml + +fmt-toml/%: % + toml-sort --in-place --all "$<" + taplo fmt "$<" diff --git a/README.md b/README.md new file mode 100644 index 00000000..1b4bc91b --- /dev/null +++ b/README.md @@ -0,0 +1,4 @@ +# sing-box-rules + +Describe your project here. +* License: MIT diff --git a/pyproject.toml b/pyproject.toml new file mode 100644 index 00000000..9302b961 --- /dev/null +++ b/pyproject.toml @@ -0,0 +1,31 @@ +[build-system] +build-backend = "hatchling.build" +requires = ["hatchling"] + +[project] +authors = [ + { email = "30631553+liblaf@users.noreply.github.com", name = "liblaf" }, +] +dependencies = [ + "pydantic>=2.7.3", + "requests>=2.32.3", +] +description = "Add your description here" +license = { text = "MIT" } +name = "sing-box-rules" +readme = "README.md" +requires-python = ">= 3.12" +version = "0.0.0" + +[project.scripts] +"sing-box-rules" = "sbr:main" + +[tool.hatch.build.targets.wheel] +packages = ["src/sbr"] + +[tool.hatch.metadata] +allow-direct-references = true + +[tool.rye] +dev-dependencies = [] +managed = true diff --git a/pyrightconfig.json b/pyrightconfig.json new file mode 100644 index 00000000..3f13fc2c --- /dev/null +++ b/pyrightconfig.json @@ -0,0 +1,3 @@ +{ + "typeCheckingMode": "standard" +} diff --git a/requirements-dev.lock b/requirements-dev.lock new file mode 100644 index 00000000..0151739f --- /dev/null +++ b/requirements-dev.lock @@ -0,0 +1,30 @@ +# generated by rye +# use `rye lock` or `rye sync` to update this lockfile +# +# last locked with the following flags: +# pre: false +# features: [] +# all-features: false +# with-sources: false +# generate-hashes: false + +-e file:. +annotated-types==0.7.0 + # via pydantic +certifi==2024.6.2 + # via requests +charset-normalizer==3.3.2 + # via requests +idna==3.7 + # via requests +pydantic==2.7.3 + # via sing-box-rules +pydantic-core==2.18.4 + # via pydantic +requests==2.32.3 + # via sing-box-rules +typing-extensions==4.12.2 + # via pydantic + # via pydantic-core +urllib3==2.2.1 + # via requests diff --git a/requirements.lock b/requirements.lock new file mode 100644 index 00000000..0151739f --- /dev/null +++ b/requirements.lock @@ -0,0 +1,30 @@ +# generated by rye +# use `rye lock` or `rye sync` to update this lockfile +# +# last locked with the following flags: +# pre: false +# features: [] +# all-features: false +# with-sources: false +# generate-hashes: false + +-e file:. +annotated-types==0.7.0 + # via pydantic +certifi==2024.6.2 + # via requests +charset-normalizer==3.3.2 + # via requests +idna==3.7 + # via requests +pydantic==2.7.3 + # via sing-box-rules +pydantic-core==2.18.4 + # via pydantic +requests==2.32.3 + # via sing-box-rules +typing-extensions==4.12.2 + # via pydantic + # via pydantic-core +urllib3==2.2.1 + # via requests diff --git a/scripts/build.py b/scripts/build.py new file mode 100644 index 00000000..f9a7d0e7 --- /dev/null +++ b/scripts/build.py @@ -0,0 +1,24 @@ +import re + +from sbr.geosite import Geosite +from sbr.rule_set import RuleSet + +URL_PREFIX: str = "https://github.com/MetaCubeX/meta-rules-dat/releases/download/latest" +geosite = Geosite(url=f"{URL_PREFIX}/geosite.db") + +geosite_ai: RuleSet = geosite.export("bing", "google", "openai", "perplexity") +geosite_ai.save("rule-sets/ai.srs") +geosite_emby: RuleSet = RuleSet.from_url( + "https://github.com/NotSFC/rulelist/raw/main/sing-box/Emby/Emby.json" +) +geosite_emby.save("rule-sets/emby.srs") +geosite_onedrive: RuleSet = geosite.export("onedrive") +geosite_onedrive.save("rule-sets/onedrive.srs") +geosite_proxy: RuleSet = geosite.export( + *[category for category in geosite.list() if re.match(r".*!cn$", category)] +) +geosite_proxy.save("rule-sets/proxy.srs") +geosite_cn: RuleSet = geosite.export( + *[category for category in geosite.list() if re.match(r"(.*[-@])?cn$", category)] +) +geosite_proxy.save("rule-sets/cn.srs") diff --git a/src/sbr/__init__.py b/src/sbr/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/src/sbr/__main__.py b/src/sbr/__main__.py new file mode 100644 index 00000000..e69de29b diff --git a/src/sbr/geosite.py b/src/sbr/geosite.py new file mode 100644 index 00000000..a56b25be --- /dev/null +++ b/src/sbr/geosite.py @@ -0,0 +1,71 @@ +import functools +import json +import pathlib +import subprocess +import tempfile +from concurrent.futures import ThreadPoolExecutor + +import requests + +from sbr.rule_set import RuleSet + + +class Geosite: + file: pathlib.Path + _dtemp: tempfile.TemporaryDirectory + + def __init__( + self, + file: pathlib.Path | None = None, + url: str = "https://github.com/MetaCubeX/meta-rules-dat/releases/download/latest/geosite.db", + ) -> None: + self._dtemp = tempfile.TemporaryDirectory() + if file is None: + self.file = self.dtemp / "geosite.db" + with self.file.open("wb") as fp: + resp: requests.Response = requests.get(url) + resp.raise_for_status() + fp.write(resp.content) + else: + self.file = file + + def export(self, *categories: str) -> RuleSet: + with ThreadPoolExecutor() as executor: + return sum(executor.map(self._export, categories), RuleSet()) + + @functools.cache + def list(self) -> list[str]: + proc: subprocess.CompletedProcess[str] = subprocess.run( + ["sing-box", "geosite", "--file", self.file, "list"], + stdin=subprocess.DEVNULL, + stdout=subprocess.PIPE, + text=True, + check=True, + ) + categories: list[str] = [] + for line in proc.stdout.splitlines(): + categories.append(line.split()[0]) + return categories + + @functools.cached_property + def dtemp(self) -> pathlib.Path: + return pathlib.Path(self._dtemp.name) + + @functools.lru_cache() + def _export(self, category: str) -> RuleSet: + output: pathlib.Path = self.dtemp / f"geosite-{category}.json" + subprocess.run( + [ + "sing-box", + "geosite", + "--file", + self.file, + "export", + category, + "--output", + output, + ], + stdin=subprocess.DEVNULL, + check=True, + ) + return RuleSet(**json.loads(output.read_text())) diff --git a/src/sbr/rule_set.py b/src/sbr/rule_set.py new file mode 100644 index 00000000..d85388bd --- /dev/null +++ b/src/sbr/rule_set.py @@ -0,0 +1,105 @@ +import pathlib +import subprocess +from typing import Annotated, Iterable, Literal + +import pydantic +import requests + +StrList = Annotated[ + list[str], pydantic.BeforeValidator(lambda x: [x] if isinstance(x, str) else x) +] + + +def merge(*lists: Iterable[str]) -> list[str]: + result: set[str] = set() + for lst in lists: + result.update(lst) + return list(result) + + +def diff(a: Iterable[str], b: Iterable[str]) -> list[str]: + return list(set(a) - set(b)) + + +class Rule(pydantic.BaseModel): + model_config = pydantic.ConfigDict(extra="forbid") + domain: StrList = [] + domain_suffix: StrList = [] + domain_keyword: StrList = [] + domain_regex: StrList = [] + ip_cidr: StrList = [] + + def __add__(self, other: "Rule") -> "Rule": + return Rule( + **{k: merge(getattr(self, k), getattr(other, k)) for k in self.keys()} + ) + + def __sub__(self, other: "Rule") -> "Rule": + return Rule( + **{k: diff(getattr(self, k), getattr(other, k)) for k in self.keys()} + ) + + def keys(self) -> Iterable[str]: + return self.model_fields.keys() + + def optimize(self) -> "Rule": + data: dict[str, set[str]] = {k: set(v) for k, v in self.model_dump().items()} + for d in self.domain: + if "." + d in data["domain_suffix"]: + data["domain"].remove(d) + data["domain_suffix"].remove("." + d) + data["domain_suffix"].add(d) + return Rule(**{k: list(v) for k, v in data.items()}) + + @pydantic.model_serializer + def serialize_model(self) -> dict[str, list[str]]: + data: dict[str, list[str]] = {k: getattr(self, k) for k in self.keys()} + data = {k: v for k, v in data.items() if v} + return data + + @classmethod + def sum(cls, rules: Iterable["Rule"], start: "Rule | None" = None) -> "Rule": + if start is None: + start = cls() + return sum(rules, start=start) + + +class RuleSet(pydantic.BaseModel): + model_config = pydantic.ConfigDict(extra="forbid") + version: Literal[1] = 1 + rules: list[Rule] = [] + + def __add__(self, other: "RuleSet") -> "RuleSet": + rule: Rule = Rule.sum(self.rules + other.rules) + return RuleSet(version=self.version, rules=[rule]) + + def __sub__(self, other: "RuleSet") -> "RuleSet": + self_rule: Rule = Rule.sum(self.rules) + other_rule: Rule = Rule.sum(other.rules) + return RuleSet(version=self.version, rules=[self_rule - other_rule]) + + def optimize(self) -> "RuleSet": + rule: Rule = Rule.sum(self.rules) + rule = rule.optimize() + return RuleSet(version=self.version, rules=[rule]) + + def save(self, file: pathlib.Path | str) -> None: + file = pathlib.Path(file) + if file.suffix == ".json": + file.write_text(self.model_dump_json()) + elif file.suffix == ".srs": + source_file: pathlib.Path = file.with_suffix(".json") + self.save(source_file) + subprocess.run( + ["sing-box", "rule-set", "compile", source_file, "--output", file], + stdin=subprocess.DEVNULL, + check=True, + ) + else: + raise NotImplementedError + + @classmethod + def from_url(cls, url: str) -> "RuleSet": + resp: requests.Response = requests.get(url) + resp.raise_for_status() + return cls(**resp.json())