diff --git a/data/shell-completions/bash/meson b/data/shell-completions/bash/meson index 404369818dad..7ad304bd5b22 100644 --- a/data/shell-completions/bash/meson +++ b/data/shell-completions/bash/meson @@ -715,6 +715,7 @@ _meson-wrap() { list promote search + set-sources status update update-db @@ -865,6 +866,28 @@ _meson-wrap-search() { COMPREPLY+=($(compgen -W '${wraps[*]}' -- "$cur")) } +_meson-wrap-set-sources() { + shortopts=( + h + ) + + longopts=( + help + ) + + local cur prev + if ! _get_comp_words_by_ref cur prev &>/dev/null; then + cur="${COMP_WORDS[COMP_CWORD]}" + fi + + if ! _meson_compgen_options "$cur"; then + if [[ -z $cur ]]; then + COMPREPLY+=($(compgen -P '--' -W '${longopts[*]}')) + COMPREPLY+=($(compgen -P '-' -W '${shortopts[*]}')) + fi + fi +} + _meson-wrap-status() { shortopts=( h diff --git a/data/shell-completions/zsh/_meson b/data/shell-completions/zsh/_meson index f64dfd4f3d44..14831f3b8175 100644 --- a/data/shell-completions/zsh/_meson +++ b/data/shell-completions/zsh/_meson @@ -271,6 +271,7 @@ _arguments \ 'update:Update a project to its newest available version' 'info:Show info about a wrap' 'status:Show the status of your subprojects' + 'set-sources:Set WrapDB source URLs' ) if (( CURRENT == 2 )); then @@ -297,6 +298,8 @@ _arguments \ # TODO: how do you figure out what wraps are provided by subprojects if # they haven't been fetched yet? _arguments '*:' + elif [[ $cmd == "set-sources" ]]; then + _arguments '*:' fi else _message "unknown meson wrap command: $words[2]" diff --git a/docs/markdown/Using-wraptool.md b/docs/markdown/Using-wraptool.md index edbceaadaec3..77db9e162d88 100644 --- a/docs/markdown/Using-wraptool.md +++ b/docs/markdown/Using-wraptool.md @@ -96,3 +96,17 @@ but available in WrapDB will automatically be downloaded. Automatic fetch of WrapDB subprojects can be disabled by removing the file `subprojects/wrapdb.json`, or by using `--wrap-mode=nodownload`. + +## Self-hosted Wrap database + +Should you wish to use a self-hosted, proxied, or an alternative Wrap database server (since version 1.X.X), you can configure server address for use with your project: + +```console +$ meson wrap set-sources https://user:password@wrapdb.mydomain.invalid:8080/subdir/ +$ meson wrap update-db +$ meson wrap install zlib +``` + +All of the following `search`, `install`, `info`, etc. wrap commands will use this address to get releases data and wrap files. +You will be limited to the wraps available on the mirror as only one source can be used at a time. +The address is stored in `subprojects/wrapdb-mirrors.json`, remove the file to use upstream server again. diff --git a/mesonbuild/wrap/wrap.py b/mesonbuild/wrap/wrap.py index 7aae1663fd1f..4e5ad169f22c 100644 --- a/mesonbuild/wrap/wrap.py +++ b/mesonbuild/wrap/wrap.py @@ -49,21 +49,59 @@ has_ssl = False REQ_TIMEOUT = 30.0 -WHITELIST_SUBDOMAIN = 'wrapdb.mesonbuild.com' +WRAPDB_UPSTREAM_HOSTNAME = 'wrapdb.mesonbuild.com' +WRAPDB_UPSTREAM_ADDRESS = f'https://{WRAPDB_UPSTREAM_HOSTNAME}' ALL_TYPES = ['file', 'git', 'hg', 'svn', 'redirect'] PATCH = shutil.which('patch') -def whitelist_wrapdb(urlstr: str) -> urllib.parse.ParseResult: - """ raises WrapException if not whitelisted subdomain """ +@lru_cache(maxsize=None) +def wrapdb_url() -> str: + try: + with Path('subprojects/wrapdb-sources.json').open('r', encoding='utf-8') as f: + config = json.load(f) + version = config['version'] + if version > 1: + m = f'WrapDB sources file (v{version}) was created with a newer version of meson.' + raise WrapException(m) + source = str(config['sources'][0]) + url = urllib.parse.urlparse(source) + if not url.scheme: + m = f'WrapDB source address requires a protocol scheme, like `{WRAPDB_UPSTREAM_ADDRESS}`.' + raise WrapException(m) + return source + except FileNotFoundError: + return WRAPDB_UPSTREAM_ADDRESS + +def is_wrapdb_subdomain(hostname: str) -> bool: + trusted_subdomains = { + WRAPDB_UPSTREAM_HOSTNAME, + urllib.parse.urlparse(wrapdb_url()).hostname, + } + for entry in trusted_subdomains: + if hostname.endswith(entry): + return True + + return False + +def expand_wrapdburl(urlstr: str, allow_insecure: bool = False) -> urllib.parse.ParseResult: url = urllib.parse.urlparse(urlstr) + if url.scheme == 'wrapdb': + if url.netloc: + raise WrapException(f'{urlstr} with wrapdb: scheme should not have a netloc') + # append wrapdb path on top of the source address + rel_path = url.path.lstrip('/') + url = urllib.parse.urlparse(urllib.parse.urljoin(wrapdb_url(), rel_path)) + if not url.hostname: - raise WrapException(f'{urlstr} is not a valid URL') - if not url.hostname.endswith(WHITELIST_SUBDOMAIN): - raise WrapException(f'{urlstr} is not a whitelisted WrapDB URL') - if has_ssl and not url.scheme == 'https': - raise WrapException(f'WrapDB did not have expected SSL https url, instead got {urlstr}') + if url.scheme not in {'file'}: + raise WrapException(f'{urlstr} is not a valid URL') + else: + if not is_wrapdb_subdomain(url.hostname): + raise WrapException(f'{urlstr} is not a whitelisted WrapDB URL') + if has_ssl and not allow_insecure and url.scheme not in {'https', 'ftps'}: + raise WrapException(f'WrapDB did not have expected SSL url, instead got {urllib.parse.urlunparse(url)}') return url def open_wrapdburl(urlstring: str, allow_insecure: bool = False, have_opt: bool = False) -> 'http.client.HTTPResponse': @@ -72,12 +110,13 @@ def open_wrapdburl(urlstring: str, allow_insecure: bool = False, have_opt: bool else: insecure_msg = '' - url = whitelist_wrapdb(urlstring) + url = expand_wrapdburl(urlstring, allow_insecure) if has_ssl: + urlstring_ = urllib.parse.urlunparse(url) try: - return T.cast('http.client.HTTPResponse', urllib.request.urlopen(urllib.parse.urlunparse(url), timeout=REQ_TIMEOUT)) + return T.cast('http.client.HTTPResponse', urllib.request.urlopen(urlstring_, timeout=REQ_TIMEOUT)) except OSError as excp: - msg = f'WrapDB connection failed to {urlstring} with error {excp}.' + msg = f'WrapDB connection failed to {urlstring_} with error {excp}.' if isinstance(excp, urllib.error.URLError) and isinstance(excp.reason, ssl.SSLCertVerificationError): if allow_insecure: mlog.warning(f'{msg}\n\n Proceeding without authentication.') @@ -92,15 +131,21 @@ def open_wrapdburl(urlstring: str, allow_insecure: bool = False, have_opt: bool mlog.warning(f'SSL module not available in {sys.executable}: WrapDB traffic not authenticated.', once=True) # If we got this far, allow_insecure was manually passed - nossl_url = url._replace(scheme='http') + if has_ssl: + ctx = ssl.create_default_context() + ctx.check_hostname = False + ctx.verify_mode = ssl.CERT_NONE + else: + msg = f'Fix python installation or change WrapDB source address to an insecure alternative, e.g. `meson wrap set-sources http://{WRAPDB_UPSTREAM_HOSTNAME}`' + raise WrapException(f'SSL protocol requested: {msg}') try: - return T.cast('http.client.HTTPResponse', urllib.request.urlopen(urllib.parse.urlunparse(nossl_url), timeout=REQ_TIMEOUT)) + return T.cast('http.client.HTTPResponse', urllib.request.urlopen(urlstring_, timeout=REQ_TIMEOUT, context=ctx)) except OSError as excp: - raise WrapException(f'WrapDB connection failed to {urlstring} with error {excp}') + raise WrapException(f'WrapDB connection failed to {urlstring_} with error {excp}') def get_releases_data(allow_insecure: bool) -> bytes: - url = open_wrapdburl('https://wrapdb.mesonbuild.com/v2/releases.json', allow_insecure, True) - return url.read() + resp = open_wrapdburl('wrapdb:///v2/releases.json', allow_insecure, True) + return resp.read() @lru_cache(maxsize=None) def get_releases(allow_insecure: bool) -> T.Dict[str, T.Any]: @@ -108,14 +153,14 @@ def get_releases(allow_insecure: bool) -> T.Dict[str, T.Any]: return T.cast('T.Dict[str, T.Any]', json.loads(data.decode())) def update_wrap_file(wrapfile: str, name: str, new_version: str, new_revision: str, allow_insecure: bool) -> None: - url = open_wrapdburl(f'https://wrapdb.mesonbuild.com/v2/{name}_{new_version}-{new_revision}/{name}.wrap', - allow_insecure, True) + resp = open_wrapdburl(f'wrapdb:///v2/{name}_{new_version}-{new_revision}/{name}.wrap', + allow_insecure, True) with open(wrapfile, 'wb') as f: - f.write(url.read()) + f.write(resp.read()) def parse_patch_url(patch_url: str) -> T.Tuple[str, str]: u = urllib.parse.urlparse(patch_url) - if u.netloc != 'wrapdb.mesonbuild.com': + if not is_wrapdb_subdomain(u.hostname): raise WrapException(f'URL {patch_url} does not seems to be a wrapdb patch') arr = u.path.strip('/').split('/') if arr[0] == 'v1': @@ -384,10 +429,10 @@ def get_from_wrapdb(self, subp_name: str) -> T.Optional[PackageDefinition]: self.check_can_download() latest_version = info['versions'][0] version, revision = latest_version.rsplit('-', 1) - url = urllib.request.urlopen(f'https://wrapdb.mesonbuild.com/v2/{subp_name}_{version}-{revision}/{subp_name}.wrap') + resp = open_wrapdburl(f'wrapdb:///v2/{subp_name}_{version}-{revision}/{subp_name}.wrap') fname = Path(self.subdir_root, f'{subp_name}.wrap') with fname.open('wb') as f: - f.write(url.read()) + f.write(resp.read()) mlog.log(f'Installed {subp_name} version {version} revision {revision}') wrap = PackageDefinition.from_wrap_file(str(fname)) self.wraps[wrap.name] = wrap @@ -687,9 +732,9 @@ def get_data(self, urlstring: str) -> T.Tuple[str, str]: h = hashlib.sha256() tmpfile = tempfile.NamedTemporaryFile(mode='wb', dir=self.cachedir, delete=False) url = urllib.parse.urlparse(urlstring) - if url.hostname and url.hostname.endswith(WHITELIST_SUBDOMAIN): + if url.hostname and is_wrapdb_subdomain(url.hostname): resp = open_wrapdburl(urlstring, allow_insecure=self.allow_insecure, have_opt=self.wrap_frontend) - elif WHITELIST_SUBDOMAIN in urlstring: + elif WRAPDB_UPSTREAM_HOSTNAME in urlstring: raise WrapException(f'{urlstring} may be a WrapDB-impersonating URL') else: headers = { diff --git a/mesonbuild/wrap/wraptool.py b/mesonbuild/wrap/wraptool.py index 5486a26a7782..8dc2341200df 100644 --- a/mesonbuild/wrap/wraptool.py +++ b/mesonbuild/wrap/wraptool.py @@ -7,6 +7,7 @@ import configparser import shutil import typing as T +import json from glob import glob from .wrap import (open_wrapdburl, WrapException, get_releases, get_releases_data, @@ -59,6 +60,11 @@ def add_arguments(parser: 'argparse.ArgumentParser') -> None: p.add_argument('project_path') p.set_defaults(wrap_func=promote) + p = subparsers.add_parser('set-sources', help='Set WrapDB source URLs (Since 1.X.X)') + p.add_argument('source', default=None, nargs='+', + help='WrapDB source URL') + p.set_defaults(wrap_func=set_db_sources) + p = subparsers.add_parser('update-db', help='Update list of projects available in WrapDB (Since 0.61.0)') p.add_argument('--allow-insecure', default=False, action='store_true', help='Allow insecure server connections.') @@ -99,7 +105,7 @@ def install(options: 'argparse.Namespace') -> None: if os.path.exists(wrapfile): raise SystemExit('Wrap file already exists.') (version, revision) = get_latest_version(name, options.allow_insecure) - url = open_wrapdburl(f'https://wrapdb.mesonbuild.com/v2/{name}_{version}-{revision}/{name}.wrap', options.allow_insecure, True) + url = open_wrapdburl(f'wrapdb:///v2/{name}_{version}-{revision}/{name}.wrap', options.allow_insecure, True) with open(wrapfile, 'wb') as f: f.write(url.read()) print(f'Installed {name} version {version} revision {revision}') @@ -187,6 +193,15 @@ def status(options: 'argparse.Namespace') -> None: else: print('', name, f'not up to date. Have {current_branch} {current_revision}, but {latest_branch} {latest_revision} is available.') +def set_db_sources(options: 'argparse.Namespace') -> None: + Path('subprojects').mkdir(exist_ok=True) + with Path('subprojects/wrapdb-sources.json').open('w', encoding='utf-8') as f: + json.dump({ + # don't guess how to preserve compatibility if stored format changes + "version": 1, + "sources": options.source, + }, f) + def update_db(options: 'argparse.Namespace') -> None: data = get_releases_data(options.allow_insecure) Path('subprojects').mkdir(exist_ok=True)