diff --git a/.github/workflows/list-deploy.yml b/.github/workflows/list-deploy.yml index 8606a04..97bff5e 100644 --- a/.github/workflows/list-deploy.yml +++ b/.github/workflows/list-deploy.yml @@ -1,9 +1,14 @@ on: + pull_request: + paths: + - mirrors.yaml + - generate-site.py push: branches: - - 'master' + - master paths: - - 'mirrors.lst' + - mirrors.yaml + - generate-site.py name: Deploy to Github Pages @@ -17,15 +22,18 @@ jobs: runs-on: ubuntu-latest steps: - name: Checkout - uses: actions/checkout@v3 + uses: classabbyamp/treeless-checkout-action@v1 + - name: Prepare + run: python3 -m venv env && env/bin/pip install PyYAML - name: Create file structure - run: make deploy + run: PYTHON=env/bin/python make deploy - name: Upload artifact uses: actions/upload-pages-artifact@v1 deploy: name: Deploy runs-on: ubuntu-latest needs: build + if: github.event == 'push' permissions: pages: write id-token: write diff --git a/Makefile b/Makefile index 4960b56..baae86a 100644 --- a/Makefile +++ b/Makefile @@ -2,6 +2,7 @@ VERSION ?= 0.3 DESTDIR ?= PREFIX ?= /usr/local MIRRORURL ?= https://xmirror.voidlinux.org/raw/mirrors.lst +PYTHON ?= python3 .PHONY: all completions install clean deploy @@ -31,13 +32,5 @@ clean: README: xmirror.1 mandoc -Tutf8 $< | col -bx >$@ -deploy: mirrors.lst - mkdir -p _site/raw - # a hacky but simple homepage that redirects to the manpage - @echo 'generating redirect page at _site/index.html' - @printf '> _site/index.html - @printf '" />' >> _site/index.html - @printf '

Redirect

' >> _site/index.html - @printf '\n' >> _site/index.html - cp mirrors.lst _site/raw/ +deploy: mirrors.yaml + $(PYTHON) generate-site.py mirrors.yaml _site diff --git a/generate-site.py b/generate-site.py new file mode 100644 index 0000000..9095b85 --- /dev/null +++ b/generate-site.py @@ -0,0 +1,191 @@ +#!/usr/bin/python3 + +import csv +import json +import re +from pathlib import Path +from shutil import rmtree +from sys import argv, stderr + +import yaml + + +html = """ +

Void Linux Mirrors

+

Void Linux maintains mirrors in several geographic regions for users. A fresh +install will default to +repo-default.voidlinux.org, +which may map to any Tier 1 mirror, but you may have a better experience by +selecting +a different mirror.

+ +

The status of all listed mirrors can be viewed on Void's +Grafana dashboard.

+ +

Tier 1 mirrors

+

Tier 1 mirrors are maintained by the Void Linux Infrastructure Team. These +mirrors sync directly from the build servers and will always have the latest +packages available.

+ +{tier1} + +

Tier 2 mirrors

+

Tier 2 mirrors sync from a nearby Tier 1 mirror when possible. These mirrors are +not managed by Void and do not have any guarantees of freshness or completeness +of packages, nor are they required to sync every available architecture or +sub-repository.

+ +{tier2} + +

Tor Mirrors

+

Void Linux is also mirrored on the Tor network. See +Using Tor Mirrors +for more information.

+ +

Creating a Mirror

+

If you'd like to set up a mirror, and are confident you can keep it reasonably +up-to-date, follow one of the many guides available for mirroring with +rsync(1). You should be syncing from +rsync://repo-sync.voidlinux.org/voidlinux/. To list your mirror on this site, +submit a pull request to +the xmirror repository that adds your +mirror to mirrors.yaml.

+

A full mirror requires around 1TB of storage. It is also possible to mirror only +part of the repositories. Excluding debug packages is one way of decreasing the +load on the Tier 1 mirrors, with low impact on users.

+

Please keep in mind that we pay bandwidth for all data sent out from the Tier 1 +mirrors. You can respect this by only mirroring if your use case for your mirror +will offset the network throughput consumed by your mirror syncing.

+""" + +tabletmpl = """ + + + + + + +{rows} +
MirrorRegionLocation
+""" + +rowtmpl = """ + {url} + {region} + {location} + +""" + + +regions = { + "AF": "Africa", + "AN": "Antarctica", + "AS": "Asia", + "EU": "Europe", + "NA": "North America", + "OC": "Oceania", + "SA": "South and Central America", + "World": "Globally Available", +} + + +def argparse() -> tuple[Path, Path]: + if len(argv) != 3: + print(f"usage: {argv[0]} ", file=stderr) + raise SystemExit(1) + + srcfile = Path(argv[1]) + if not srcfile.is_file(): + print(f"{argv[0]}: {srcfile} does not exist or is not a file", file=stderr) + print(f"usage: {argv[0]} ", file=stderr) + raise SystemExit(1) + + destdir = Path(argv[2]) + if destdir.exists() and not destdir.is_dir(): + print(f"{argv[0]}: {destdir} exists but is not a directory", file=stderr) + print(f"usage: {argv[0]} ", file=stderr) + raise SystemExit(1) + + return (srcfile, destdir) + + +def write_api(destdir: Path, data: list[dict]): + apidir = destdir / "v0" + apidir.mkdir(parents=True) + with (apidir / "mirrors.json").open("w") as f: + json.dump(data, f) + + +def write_metrics(destdir: Path, data: list[dict]): + """ + generate files for assessing mirror status with prometheus + """ + def prom_targets(data: list[dict]) -> list[dict]: + """ + transforms the mirror list into a prometheus target list format + https://prometheus.io/docs/prometheus/latest/http_sd/ + """ + tgts = sorted(re.sub(r"^(?:http|ftp)s?://(.*?)/?$", r"\1/current", m["base_url"]) for m in data) + return [{"targets": tgts}] + + metricsdir = destdir / "metrics" + metricsdir.mkdir(parents=True) + promdata = prom_targets(data) + with (metricsdir / "prometheus.json").open("w") as f: + json.dump(promdata, f) + +def write_raw(destdir: Path, data: list[dict]): + """ + generate the tab-separated value file for bash xmirror + """ + rawdir = destdir / "raw" + rawdir.mkdir(parents=True) + with (rawdir / "mirrors.lst").open("w") as f: + wr = csv.writer(f, dialect="excel-tab") + wr.writerows([[m["region"], m["base_url"], m["location"], m["tier"]] for m in data if m["enabled"]]) + + +def write_html(destdir: Path, data: list[dict]): + """ + write the list of mirrors into 2 tables, one for each tier, and insert them in the html document + """ + def reg(r: str) -> str: + """get a pretty name for a region""" + if r in regions.keys(): + return regions[r] + return r + + tier1 = tabletmpl.format(rows="\n".join( + [rowtmpl.format(url=m["base_url"], region=reg(m["region"]), location=m["location"]) + for m in data if m["tier"] == 1 and m["enabled"]] + )) + tier2 = tabletmpl.format(rows="\n".join( + [rowtmpl.format(url=m["base_url"], region=reg(m["region"]), location=m["location"]) + for m in data if m["tier"] == 2 and m["enabled"]] + )) + with (destdir / "index.html").open("w") as f: + f.write(html.format(tier1=tier1, tier2=tier2)) + + +if __name__ == "__main__": + srcfile, destdir = argparse() + + # clean up + if destdir.exists(): + rmtree(destdir) + + # load data + with srcfile.open() as f: + data = yaml.safe_load(f.read()) + + # write v0 api json + write_api(destdir, data) + + # write prometheus json + write_metrics(destdir, data) + + # write raw tsv data + write_raw(destdir, data) + + # write html table + write_html(destdir, data) diff --git a/mirrors.lst b/mirrors.lst deleted file mode 100644 index c391ef7..0000000 --- a/mirrors.lst +++ /dev/null @@ -1,36 +0,0 @@ -# see xmirror(1) for more details about this file -# region url location tier -####### Tier 1 -EU https://repo-fi.voidlinux.org/ Helsinki, Finland 1 -EU https://repo-de.voidlinux.org/ Frankfurt, Germany 1 -NA https://mirrors.servercentral.com/voidlinux/ Chicago, USA 1 -#NA https://repo-us.voidlinux.org/ Kansas City, USA 1 -World https://repo-fastly.voidlinux.org/ Fastly Global CDN 1 -####### Tier 2 -AS https://mirror.ps.kz/voidlinux/ Almaty, Kazakhstan 2 -AS https://mirror.nju.edu.cn/voidlinux/ China 2 -AS https://mirrors.bfsu.edu.cn/voidlinux/ Beijing, China 2 -AS https://mirrors.cnnic.cn/voidlinux/ Beijing, China 2 -AS https://mirrors.tuna.tsinghua.edu.cn/voidlinux/ Beijing, China 2 -AS https://mirror.sjtu.edu.cn/voidlinux/ Shanghai, China 2 -AS https://repo.jing.rocks/voidlinux/ Tokyo, Japan 2 -AS https://void.webconverger.org/ Singapore 2 -EU http://ftp.dk.xemacs.org/voidlinux/ Denmark 2 -EU https://mirrors.dotsrc.org/voidlinux/ Denmark 2 -EU https://ftp.cc.uoc.gr/mirrors/linux/voidlinux/ Greece 2 -EU https://quantum-mirror.hu/mirrors/pub/voidlinux/ Hungary 2 -EU https://voidlinux.mirror.garr.it/ Italy 2 -EU https://void.cijber.net/ Amsterdam, Netherlands 2 -EU https://void.sakamoto.pl/ Warsaw, Poland 2 -EU http://ftp.debian.ru/mirrors/voidlinux/ Russia 2 -EU https://mirror.yandex.ru/mirrors/voidlinux/ Russia 2 -EU https://ftp.lysator.liu.se/pub/voidlinux/ Sweden 2 -EU https://mirror.accum.se/mirror/voidlinux/ Sweden 2 -EU https://mirror.puzzle.ch/voidlinux/ Bern, Switzerland 2 -NA https://mirror.vofr.net/voidlinux/ California, USA 2 -NA https://mirror2.sandyriver.net/pub/voidlinux/ Kentucky, USA 2 -NA https://mirror.clarkson.edu/voidlinux/ New York, USA 2 -OC https://mirror.aarnet.edu.au/pub/voidlinux/ Canberra, Australia 2 -OC https://ftp.swin.edu.au/voidlinux/ Melbourne, Australia 2 -SA https://voidlinux.com.br/repo/ Ouro Preto, Brazil 2 -SA http://void.chililinux.com/voidlinux/ Pimenta Bueno, Brazil 2 diff --git a/mirrors.yaml b/mirrors.yaml new file mode 100644 index 0000000..75435c9 --- /dev/null +++ b/mirrors.yaml @@ -0,0 +1,160 @@ +- base_url: https://repo-fi.voidlinux.org/ + region: EU + location: Helsinki, Finland + tier: 1 + enabled: true +- base_url: https://repo-de.voidlinux.org/ + region: EU + location: Frankfurt, Germany + tier: 1 + enabled: true +- base_url: https://repo-us.voidlinux.org/ + region: NA + location: Kansas City, USA + tier: 1 + enabled: false +- base_url: https://repo-fastly.voidlinux.org/ + region: World + location: Fastly Global CDN + tier: 1 + enabled: true +- base_url: https://mirrors.servercentral.com/voidlinux/ + region: NA + location: Chicago, USA + tier: 1 + enabled: true +- base_url: https://mirror.ps.kz/voidlinux/ + region: AS + location: Almaty, Kazakhstan + tier: 2 + enabled: true +- base_url: https://mirror.nju.edu.cn/voidlinux/ + region: AS + location: China + tier: 2 + enabled: true +- base_url: https://mirrors.bfsu.edu.cn/voidlinux/ + region: AS + location: Beijing, China + tier: 2 + enabled: true +- base_url: https://mirrors.cnnic.cn/voidlinux/ + region: AS + location: Beijing, China + tier: 2 + enabled: true +- base_url: https://mirrors.tuna.tsinghua.edu.cn/voidlinux/ + region: AS + location: Beijing, China + tier: 2 + enabled: true +- base_url: https://mirror.sjtu.edu.cn/voidlinux/ + region: AS + location: Shanghai, China + tier: 2 + enabled: true +- base_url: https://repo.jing.rocks/voidlinux/ + region: AS + location: Tokyo, Japan + tier: 2 + enabled: true +- base_url: https://void.webconverger.org/ + region: AS + location: Singapore + tier: 2 + enabled: true +- base_url: http://ftp.dk.xemacs.org/voidlinux/ + region: EU + location: Denmark + tier: 2 + enabled: true +- base_url: https://mirrors.dotsrc.org/voidlinux/ + region: EU + location: Denmark + tier: 2 + enabled: true +- base_url: https://ftp.cc.uoc.gr/mirrors/linux/voidlinux/ + region: EU + location: Greece + tier: 2 + enabled: true +- base_url: https://quantum-mirror.hu/mirrors/pub/voidlinux/ + region: EU + location: Hungary + tier: 2 + enabled: true +- base_url: https://voidlinux.mirror.garr.it/ + region: EU + location: Italy + tier: 2 + enabled: true +- base_url: https://void.cijber.net/ + region: EU + location: Amsterdam, Netherlands + tier: 2 + enabled: true +- base_url: https://void.sakamoto.pl/ + region: EU + location: Warsaw, Poland + tier: 2 + enabled: true +- base_url: http://ftp.debian.ru/mirrors/voidlinux/ + region: EU + location: Russia + tier: 2 + enabled: true +- base_url: https://mirror.yandex.ru/mirrors/voidlinux/ + region: EU + location: Russia + tier: 2 + enabled: true +- base_url: https://ftp.lysator.liu.se/pub/voidlinux/ + region: EU + location: Sweden + tier: 2 + enabled: true +- base_url: https://mirror.accum.se/mirror/voidlinux/ + region: EU + location: Sweden + tier: 2 + enabled: true +- base_url: https://mirror.puzzle.ch/voidlinux/ + region: EU + location: Bern, Switzerland + tier: 2 + enabled: true +- base_url: https://mirror.vofr.net/voidlinux/ + region: NA + location: California, USA + tier: 2 + enabled: true +- base_url: https://mirror2.sandyriver.net/pub/voidlinux/ + region: NA + location: Kentucky, USA + tier: 2 + enabled: true +- base_url: https://mirror.clarkson.edu/voidlinux/ + region: NA + location: New York, USA + tier: 2 + enabled: true +- base_url: https://mirror.aarnet.edu.au/pub/voidlinux/ + region: OC + location: Canberra, Australia + tier: 2 + enabled: true +- base_url: https://ftp.swin.edu.au/voidlinux/ + region: OC + location: Melbourne, Australia + tier: 2 + enabled: true +- base_url: https://voidlinux.com.br/repo/ + region: SA + location: Ouro Preto, Brazil + tier: 2 + enabled: true +- base_url: http://void.chililinux.com/voidlinux/ + region: SA + location: Pimenta Bueno, Brazil + tier: 2 + enabled: true