From 93b91007fe26cf3479e3e9f73ac7a3f7d1308c72 Mon Sep 17 00:00:00 2001 From: classabbyamp Date: Wed, 25 Oct 2023 06:01:18 -0400 Subject: [PATCH] generate a more complete website, with json, tsv, and more this centralises all mirror information to one page, and allows for more easily setting up new mirrors, because the mirror information, xmirror data files, and prometheus configurationo can be generated at onces. --- .github/workflows/list-deploy.yml | 11 +- Makefile | 13 +- generate-site.py | 191 ++++++++++++++++++++++++++++++ mirrors.lst | 36 ------ mirrors.yaml | 160 +++++++++++++++++++++++++ 5 files changed, 360 insertions(+), 51 deletions(-) create mode 100644 generate-site.py delete mode 100644 mirrors.lst create mode 100644 mirrors.yaml diff --git a/.github/workflows/list-deploy.yml b/.github/workflows/list-deploy.yml index 8606a04..973ebdd 100644 --- a/.github/workflows/list-deploy.yml +++ b/.github/workflows/list-deploy.yml @@ -1,9 +1,7 @@ on: push: - branches: - - 'master' paths: - - 'mirrors.lst' + - mirrors.yaml name: Deploy to Github Pages @@ -17,15 +15,18 @@ jobs: runs-on: ubuntu-latest steps: - name: Checkout - uses: actions/checkout@v3 + uses: classabbyamp/treeless-checkout-action@v1 + - name: Prepare + run: python3 -m venv env && env/bin/pip install PyYAML - name: Create file structure - run: make deploy + run: PYTHON=env/bin/python make deploy - name: Upload artifact uses: actions/upload-pages-artifact@v1 deploy: name: Deploy runs-on: ubuntu-latest needs: build + if: github.ref_name == 'master' permissions: pages: write id-token: write diff --git a/Makefile b/Makefile index 4960b56..baae86a 100644 --- a/Makefile +++ b/Makefile @@ -2,6 +2,7 @@ VERSION ?= 0.3 DESTDIR ?= PREFIX ?= /usr/local MIRRORURL ?= https://xmirror.voidlinux.org/raw/mirrors.lst +PYTHON ?= python3 .PHONY: all completions install clean deploy @@ -31,13 +32,5 @@ clean: README: xmirror.1 mandoc -Tutf8 $< | col -bx >$@ -deploy: mirrors.lst - mkdir -p _site/raw - # a hacky but simple homepage that redirects to the manpage - @echo 'generating redirect page at _site/index.html' - @printf '> _site/index.html - @printf '" />' >> _site/index.html - @printf '

Redirect

' >> _site/index.html - @printf '\n' >> _site/index.html - cp mirrors.lst _site/raw/ +deploy: mirrors.yaml + $(PYTHON) generate-site.py mirrors.yaml _site diff --git a/generate-site.py b/generate-site.py new file mode 100644 index 0000000..9095b85 --- /dev/null +++ b/generate-site.py @@ -0,0 +1,191 @@ +#!/usr/bin/python3 + +import csv +import json +import re +from pathlib import Path +from shutil import rmtree +from sys import argv, stderr + +import yaml + + +html = """ +

Void Linux Mirrors

+

Void Linux maintains mirrors in several geographic regions for users. A fresh +install will default to +repo-default.voidlinux.org, +which may map to any Tier 1 mirror, but you may have a better experience by +selecting +a different mirror.

+ +

The status of all listed mirrors can be viewed on Void's +Grafana dashboard.

+ +

Tier 1 mirrors

+

Tier 1 mirrors are maintained by the Void Linux Infrastructure Team. These +mirrors sync directly from the build servers and will always have the latest +packages available.

+ +{tier1} + +

Tier 2 mirrors

+

Tier 2 mirrors sync from a nearby Tier 1 mirror when possible. These mirrors are +not managed by Void and do not have any guarantees of freshness or completeness +of packages, nor are they required to sync every available architecture or +sub-repository.

+ +{tier2} + +

Tor Mirrors

+

Void Linux is also mirrored on the Tor network. See +Using Tor Mirrors +for more information.

+ +

Creating a Mirror

+

If you'd like to set up a mirror, and are confident you can keep it reasonably +up-to-date, follow one of the many guides available for mirroring with +rsync(1). You should be syncing from +rsync://repo-sync.voidlinux.org/voidlinux/. To list your mirror on this site, +submit a pull request to +the xmirror repository that adds your +mirror to mirrors.yaml.

+

A full mirror requires around 1TB of storage. It is also possible to mirror only +part of the repositories. Excluding debug packages is one way of decreasing the +load on the Tier 1 mirrors, with low impact on users.

+

Please keep in mind that we pay bandwidth for all data sent out from the Tier 1 +mirrors. You can respect this by only mirroring if your use case for your mirror +will offset the network throughput consumed by your mirror syncing.

+""" + +tabletmpl = """ + + + + + + +{rows} +
MirrorRegionLocation
+""" + +rowtmpl = """ + {url} + {region} + {location} + +""" + + +regions = { + "AF": "Africa", + "AN": "Antarctica", + "AS": "Asia", + "EU": "Europe", + "NA": "North America", + "OC": "Oceania", + "SA": "South and Central America", + "World": "Globally Available", +} + + +def argparse() -> tuple[Path, Path]: + if len(argv) != 3: + print(f"usage: {argv[0]} ", file=stderr) + raise SystemExit(1) + + srcfile = Path(argv[1]) + if not srcfile.is_file(): + print(f"{argv[0]}: {srcfile} does not exist or is not a file", file=stderr) + print(f"usage: {argv[0]} ", file=stderr) + raise SystemExit(1) + + destdir = Path(argv[2]) + if destdir.exists() and not destdir.is_dir(): + print(f"{argv[0]}: {destdir} exists but is not a directory", file=stderr) + print(f"usage: {argv[0]} ", file=stderr) + raise SystemExit(1) + + return (srcfile, destdir) + + +def write_api(destdir: Path, data: list[dict]): + apidir = destdir / "v0" + apidir.mkdir(parents=True) + with (apidir / "mirrors.json").open("w") as f: + json.dump(data, f) + + +def write_metrics(destdir: Path, data: list[dict]): + """ + generate files for assessing mirror status with prometheus + """ + def prom_targets(data: list[dict]) -> list[dict]: + """ + transforms the mirror list into a prometheus target list format + https://prometheus.io/docs/prometheus/latest/http_sd/ + """ + tgts = sorted(re.sub(r"^(?:http|ftp)s?://(.*?)/?$", r"\1/current", m["base_url"]) for m in data) + return [{"targets": tgts}] + + metricsdir = destdir / "metrics" + metricsdir.mkdir(parents=True) + promdata = prom_targets(data) + with (metricsdir / "prometheus.json").open("w") as f: + json.dump(promdata, f) + +def write_raw(destdir: Path, data: list[dict]): + """ + generate the tab-separated value file for bash xmirror + """ + rawdir = destdir / "raw" + rawdir.mkdir(parents=True) + with (rawdir / "mirrors.lst").open("w") as f: + wr = csv.writer(f, dialect="excel-tab") + wr.writerows([[m["region"], m["base_url"], m["location"], m["tier"]] for m in data if m["enabled"]]) + + +def write_html(destdir: Path, data: list[dict]): + """ + write the list of mirrors into 2 tables, one for each tier, and insert them in the html document + """ + def reg(r: str) -> str: + """get a pretty name for a region""" + if r in regions.keys(): + return regions[r] + return r + + tier1 = tabletmpl.format(rows="\n".join( + [rowtmpl.format(url=m["base_url"], region=reg(m["region"]), location=m["location"]) + for m in data if m["tier"] == 1 and m["enabled"]] + )) + tier2 = tabletmpl.format(rows="\n".join( + [rowtmpl.format(url=m["base_url"], region=reg(m["region"]), location=m["location"]) + for m in data if m["tier"] == 2 and m["enabled"]] + )) + with (destdir / "index.html").open("w") as f: + f.write(html.format(tier1=tier1, tier2=tier2)) + + +if __name__ == "__main__": + srcfile, destdir = argparse() + + # clean up + if destdir.exists(): + rmtree(destdir) + + # load data + with srcfile.open() as f: + data = yaml.safe_load(f.read()) + + # write v0 api json + write_api(destdir, data) + + # write prometheus json + write_metrics(destdir, data) + + # write raw tsv data + write_raw(destdir, data) + + # write html table + write_html(destdir, data) diff --git a/mirrors.lst b/mirrors.lst deleted file mode 100644 index c391ef7..0000000 --- a/mirrors.lst +++ /dev/null @@ -1,36 +0,0 @@ -# see xmirror(1) for more details about this file -# region url location tier -####### Tier 1 -EU https://repo-fi.voidlinux.org/ Helsinki, Finland 1 -EU https://repo-de.voidlinux.org/ Frankfurt, Germany 1 -NA https://mirrors.servercentral.com/voidlinux/ Chicago, USA 1 -#NA https://repo-us.voidlinux.org/ Kansas City, USA 1 -World https://repo-fastly.voidlinux.org/ Fastly Global CDN 1 -####### Tier 2 -AS https://mirror.ps.kz/voidlinux/ Almaty, Kazakhstan 2 -AS https://mirror.nju.edu.cn/voidlinux/ China 2 -AS https://mirrors.bfsu.edu.cn/voidlinux/ Beijing, China 2 -AS https://mirrors.cnnic.cn/voidlinux/ Beijing, China 2 -AS https://mirrors.tuna.tsinghua.edu.cn/voidlinux/ Beijing, China 2 -AS https://mirror.sjtu.edu.cn/voidlinux/ Shanghai, China 2 -AS https://repo.jing.rocks/voidlinux/ Tokyo, Japan 2 -AS https://void.webconverger.org/ Singapore 2 -EU http://ftp.dk.xemacs.org/voidlinux/ Denmark 2 -EU https://mirrors.dotsrc.org/voidlinux/ Denmark 2 -EU https://ftp.cc.uoc.gr/mirrors/linux/voidlinux/ Greece 2 -EU https://quantum-mirror.hu/mirrors/pub/voidlinux/ Hungary 2 -EU https://voidlinux.mirror.garr.it/ Italy 2 -EU https://void.cijber.net/ Amsterdam, Netherlands 2 -EU https://void.sakamoto.pl/ Warsaw, Poland 2 -EU http://ftp.debian.ru/mirrors/voidlinux/ Russia 2 -EU https://mirror.yandex.ru/mirrors/voidlinux/ Russia 2 -EU https://ftp.lysator.liu.se/pub/voidlinux/ Sweden 2 -EU https://mirror.accum.se/mirror/voidlinux/ Sweden 2 -EU https://mirror.puzzle.ch/voidlinux/ Bern, Switzerland 2 -NA https://mirror.vofr.net/voidlinux/ California, USA 2 -NA https://mirror2.sandyriver.net/pub/voidlinux/ Kentucky, USA 2 -NA https://mirror.clarkson.edu/voidlinux/ New York, USA 2 -OC https://mirror.aarnet.edu.au/pub/voidlinux/ Canberra, Australia 2 -OC https://ftp.swin.edu.au/voidlinux/ Melbourne, Australia 2 -SA https://voidlinux.com.br/repo/ Ouro Preto, Brazil 2 -SA http://void.chililinux.com/voidlinux/ Pimenta Bueno, Brazil 2 diff --git a/mirrors.yaml b/mirrors.yaml new file mode 100644 index 0000000..75435c9 --- /dev/null +++ b/mirrors.yaml @@ -0,0 +1,160 @@ +- base_url: https://repo-fi.voidlinux.org/ + region: EU + location: Helsinki, Finland + tier: 1 + enabled: true +- base_url: https://repo-de.voidlinux.org/ + region: EU + location: Frankfurt, Germany + tier: 1 + enabled: true +- base_url: https://repo-us.voidlinux.org/ + region: NA + location: Kansas City, USA + tier: 1 + enabled: false +- base_url: https://repo-fastly.voidlinux.org/ + region: World + location: Fastly Global CDN + tier: 1 + enabled: true +- base_url: https://mirrors.servercentral.com/voidlinux/ + region: NA + location: Chicago, USA + tier: 1 + enabled: true +- base_url: https://mirror.ps.kz/voidlinux/ + region: AS + location: Almaty, Kazakhstan + tier: 2 + enabled: true +- base_url: https://mirror.nju.edu.cn/voidlinux/ + region: AS + location: China + tier: 2 + enabled: true +- base_url: https://mirrors.bfsu.edu.cn/voidlinux/ + region: AS + location: Beijing, China + tier: 2 + enabled: true +- base_url: https://mirrors.cnnic.cn/voidlinux/ + region: AS + location: Beijing, China + tier: 2 + enabled: true +- base_url: https://mirrors.tuna.tsinghua.edu.cn/voidlinux/ + region: AS + location: Beijing, China + tier: 2 + enabled: true +- base_url: https://mirror.sjtu.edu.cn/voidlinux/ + region: AS + location: Shanghai, China + tier: 2 + enabled: true +- base_url: https://repo.jing.rocks/voidlinux/ + region: AS + location: Tokyo, Japan + tier: 2 + enabled: true +- base_url: https://void.webconverger.org/ + region: AS + location: Singapore + tier: 2 + enabled: true +- base_url: http://ftp.dk.xemacs.org/voidlinux/ + region: EU + location: Denmark + tier: 2 + enabled: true +- base_url: https://mirrors.dotsrc.org/voidlinux/ + region: EU + location: Denmark + tier: 2 + enabled: true +- base_url: https://ftp.cc.uoc.gr/mirrors/linux/voidlinux/ + region: EU + location: Greece + tier: 2 + enabled: true +- base_url: https://quantum-mirror.hu/mirrors/pub/voidlinux/ + region: EU + location: Hungary + tier: 2 + enabled: true +- base_url: https://voidlinux.mirror.garr.it/ + region: EU + location: Italy + tier: 2 + enabled: true +- base_url: https://void.cijber.net/ + region: EU + location: Amsterdam, Netherlands + tier: 2 + enabled: true +- base_url: https://void.sakamoto.pl/ + region: EU + location: Warsaw, Poland + tier: 2 + enabled: true +- base_url: http://ftp.debian.ru/mirrors/voidlinux/ + region: EU + location: Russia + tier: 2 + enabled: true +- base_url: https://mirror.yandex.ru/mirrors/voidlinux/ + region: EU + location: Russia + tier: 2 + enabled: true +- base_url: https://ftp.lysator.liu.se/pub/voidlinux/ + region: EU + location: Sweden + tier: 2 + enabled: true +- base_url: https://mirror.accum.se/mirror/voidlinux/ + region: EU + location: Sweden + tier: 2 + enabled: true +- base_url: https://mirror.puzzle.ch/voidlinux/ + region: EU + location: Bern, Switzerland + tier: 2 + enabled: true +- base_url: https://mirror.vofr.net/voidlinux/ + region: NA + location: California, USA + tier: 2 + enabled: true +- base_url: https://mirror2.sandyriver.net/pub/voidlinux/ + region: NA + location: Kentucky, USA + tier: 2 + enabled: true +- base_url: https://mirror.clarkson.edu/voidlinux/ + region: NA + location: New York, USA + tier: 2 + enabled: true +- base_url: https://mirror.aarnet.edu.au/pub/voidlinux/ + region: OC + location: Canberra, Australia + tier: 2 + enabled: true +- base_url: https://ftp.swin.edu.au/voidlinux/ + region: OC + location: Melbourne, Australia + tier: 2 + enabled: true +- base_url: https://voidlinux.com.br/repo/ + region: SA + location: Ouro Preto, Brazil + tier: 2 + enabled: true +- base_url: http://void.chililinux.com/voidlinux/ + region: SA + location: Pimenta Bueno, Brazil + tier: 2 + enabled: true