', '', '', }
+
+# tx resources keys
+STRING_HASH_KEY = 'string_hash'
+KEY_KEY = 'key'
+SOURCE_STRING_KEY = 'source_string'
+# markdown frontmatter keys
+PERMALINK_KEY = 'permalink'
+REDIRECT_KEY = 'redirect_from'
+# comment and tag for tx source strings
+UPDATE_TAGS = '{"comment": "Added notranslate tag via curl", "tags": ["notranslate"]}'
+
+UPDATE_TAGS_LOCKED = '{"comment": "Added locked tags via curl", "tags": ["locked"]}'
+DATA_TO_URL_MAPPING = \
+{
+ 'hcl': 'Strings are taken from this page: https://www.qubes-os.org/hcl/',
+ 'download': 'Strings are taken from this page: https://www.qubes-os.org/downloads/',
+ 'experts': 'Strings are taken from this page: https://www.qubes-os.org/experts/',
+ 'home': 'Strings are taken from this page: https://www.qubes-os.org/',
+ 'index': 'Strings are taken from this page: https://www.qubes-os.org/doc/',
+ 'intro': 'Strings are taken from this page: https://www.qubes-os.org/intro/',
+ 'partners': 'Strings are taken from this page: https://www.qubes-os.org/partners/',
+ 'style_guide_content': 'Strings are taken from this page: https://www.qubes-os.org/doc/style-guide/',
+ 'teamtexts': 'Strings are taken from this page: https://www.qubes-os.org/team/',
+ 'videos': 'Strings are taken from this page: https://www.qubes-os.org/video-tours/',
+ }
+
+# transifex schema
+TX_JSON_SCHEMA = {
+ "type":"array",
+ "items":
+ [
+ {
+ "type":"object",
+ "required": ["comment", "context", "key","string_hash","reviewed","pluralized","source_string","translation"],
+ "properties":
+ {
+ "comment": {"type" : "string"},
+ "context": {"type" : "string"},
+ "key": {"type" : "string"},
+ "string_hash": {"type" : "string"},
+ "reviewed": {"type" : "boolean"},
+ "pluralized": {"type" : "boolean"},
+ "source_string": {"type" : "string"},
+ "translation": {"type" : "string"}
+ }
+ }
+ ]
+ }
+
+tagged_locked = dict()
+
+basicConfig(level=DEBUG)
+logger = getLogger(__name__)
+LOG_FILE_NAME='/tmp/tag_strings_as_locked.log'
+
+def configure_logging(logname):
+ handler = FileHandler(logname)
+ handler.setLevel(DEBUG)
+ formatter = Formatter('%(asctime)s - %(name)s - %(levelname)s - %(message)s')
+ handler.setFormatter(formatter)
+ logger.addHandler(handler)
+
+def check_reg(string, reg):
+ '''
+ check if the given string complies the given regular expression
+ string: string to check
+ reg: regular expression
+ return true if it is the case
+ '''
+
+ g = match(reg, string)
+ return g != None
+
+def manual_break():
+ prompt = input("press c + hit enter to continue: ")
+ if prompt == 'c':
+ return
+ else:
+ manual_break()
+
+def check_for_liquid_expression(item):
+ return any (item.startswith(start) and item.endswith(end) for start, end in START_END_PATTERNS.items())
+
+def get_all_original_permalinks_and_redirects(sourcenamesfiles):
+ '''
+ get the permalinks and redirects from all the original to be translated files
+ sourcenamesfiles: a file containing all the source original file names from the tx config
+ return: a set containing the original (language code is removed) permalinks and redirects
+ '''
+ sources = []
+ with iopen(sourcenamesfiles) as fp:
+ lines = fp.readlines()
+ sources = ['./'+x.split('source_file =')[1].strip() for x in lines if lines.index(x)%2==1]
+
+ perms_and_redirects = set()
+ for filepath in sources:
+ logger.debug('reading %s' % filepath)
+ md = Post
+ with iopen(filepath) as fp:
+ md = load(fp)
+ if md.get(PERMALINK_KEY) != None:
+ perms_and_redirects.add(md.get(PERMALINK_KEY))
+ else:
+ logger.error('no permalink in frontmatter for file %s' % filepath)
+ redirects = md.get(REDIRECT_KEY)
+ if redirects != None:
+ if isinstance(redirects,list):
+ for r in redirects:
+ perms_and_redirects.add(r)
+ elif isinstance(redirects,str):
+ perms_and_redirects.add(redirects)
+ else:
+ logger.error('ERROR: unexpected in redirect_from: %s' % filepath)
+ exit(1)
+ else:
+ logger.debug('no redirect_from in frontmatter for file %s' % filepath)
+ return perms_and_redirects
+
+
+
+def create_hash_and_tags_mapping(tx_resources, tx_api_token, debug, perms_and_redirects, manual):
+ '''
+ for every given file uploaded to transifex, get its resource strings, check if they need to be marked as locked and
+ save their hashes for further processing if this is the case
+ tx_resources: a list containing all the uploaded transifex files
+ tx_api_token: transifex API token
+ debug: if true be verbose
+ manual: if true the upload will be done file by file under the supervision of the developer
+ return a dictionary containing the string hash and resource name
+ '''
+
+ hash_and_tags_mapping=dict()
+ for res in tx_resources:
+ url = 'https://www.transifex.com/api/2/project/qubes/resource/' + res + '/translation/en/strings/'
+ logger.info('will fetch url %s via curl' % url)
+ if manual:
+ manual_break()
+
+ buf = BytesIO()
+ c = Curl()
+ c.setopt(c.URL, url)
+ c.setopt(c.WRITEDATA, buf)
+ c.setopt(c.CAINFO, where())
+ c.setopt(c.USERPWD, 'api:' + tx_api_token)
+ c.setopt(c.FOLLOWLOCATION, True)
+ try:
+ c.perform()
+ except error as pe:
+ logger.error("Pycurl: ", exc_info=True)
+ c.close()
+ exit(1)
+ if c.getinfo(HTTP_CODE) == 404:
+ logger.error("Following resource was not found in transifex: %s." % res)
+ logger.error("Response: %s", buf.getvalue())
+ c.close()
+ continue
+ if c.getinfo(HTTP_CODE) != 200:
+ logger.error("Following resource could not be fetched: %s." % res)
+ logger.error("Response: %s", buf.getvalue())
+ c.close()
+ continue
+ c.close()
+
+ body = loads(buf.getvalue())
+ try:
+ validate(body, TX_JSON_SCHEMA)
+ except ValidationError as e:
+ logger.error("SEVERE! invalid json input for res: %s, url: %s" %(res, url))
+ logger.error("ValidationError: ", exc_info=True)
+ exit(1)
+
+ if debug:
+ logger.debug("___________________________")
+ logger.debug("resource strings for file %s" % res)
+
+ if isinstance(body, list):
+
+ for item in body:
+ if isinstance(item, dict):
+ if any( item[KEY_KEY] == k for k in KEY_PATTERNS ) \
+ or (item[SOURCE_STRING_KEY].startswith('![') and item[SOURCE_STRING_KEY].endswith('.png)')) \
+ or any( check_reg(item[KEY_KEY], kr) for kr in KEY_REGEX_LOCK_PATTERNS ) \
+ or any( item[SOURCE_STRING_KEY] == s for s in SOURCE_PATTERNS ) \
+ or any( item[SOURCE_STRING_KEY] == s for s in perms_and_redirects ) \
+ or any( check_reg(item[SOURCE_STRING_KEY], sr) for sr in SOURCE_REGEX_PATTERNS ) \
+ or check_for_liquid_expression( item[SOURCE_STRING_KEY] ):
+
+ hash_and_tags_mapping.update( {res + '.' + item[STRING_HASH_KEY] : res} )
+ to_debug = {res + "." + item[STRING_HASH_KEY] : [item[KEY_KEY], item[SOURCE_STRING_KEY]]}
+ tagged_locked.update(to_debug)
+ if debug:
+ logger.debug("The following resource string will be tagged as 'locked' %s" % to_debug)
+ else:
+ logger.error("got some weird stuff from transifex: %s" % item)
+ exit(1)
+ else:
+ logger.error("got some weird stuff from transifex: %s" % body)
+ exit(1)
+
+ return hash_and_tags_mapping
+
+def tag_strings_as_locked(hash_and_tag, tx_api_token, debug):
+ '''
+ upload locked tags for given string hashes for given strings for given files
+ hash_and_tag: dicitonary containing {string_hash: filename}
+ tx_api_token: transifex API token
+ debug: if true be verbose
+ '''
+
+ for stringreshash, filename in hash_and_tag.items():
+ res_hash = stringreshash.split('.')
+ stringhash = res_hash[1]
+ url = 'https://www.transifex.com/api/2/project/qubes/resource/' + filename + '/source/' + stringhash + '/'
+ logger.info('put tag %s via curl' % url)
+ buf = BytesIO()
+ c = Curl()
+ c.setopt(c.URL, url)
+ c.setopt(WRITEFUNCTION, buf.write)
+ c.setopt(c.CUSTOMREQUEST, 'PUT')
+ c.setopt(c.POST, 1)
+ c.setopt(c.USERPWD, 'api:' + tx_api_token)
+ c.setopt(c.FOLLOWLOCATION, True)
+
+ #c.setopt(c.POSTFIELDS, UPDATE_TAGS)
+ if any(filename.endswith(i) for i in DATA_TO_URL_MAPPING) and '_data_' in filename:
+ spl = filename.split('_data_')
+ comment = DATA_TO_URL_MAPPING[spl[len(spl)-1]]
+ tags = '{"comment": "' + comment + '", "tags": ["locked"]}'
+ c.setopt(c.POSTFIELDS, tags)
+ c.setopt(c.HTTPHEADER, ['Content-Type: application/json',
+ 'Content-Length: ' + str(len(tags)) ])
+ else:
+ c.setopt(c.POSTFIELDS, UPDATE_TAGS_LOCKED)
+ c.setopt(c.HTTPHEADER, ['Content-Type: application/json',
+ 'Content-Length: ' + str(len(UPDATE_TAGS_LOCKED)) ])
+ #'Content-Length: ' + str(len(UPDATE_TAGS)) ])
+
+ try:
+ c.perform()
+ except error as pe:
+ logger.error("Pycurl: ", exc_info=True)
+ c.close()
+ exit(1)
+ if c.getinfo(HTTP_CODE) == 404:
+ logger.error("Following string hash %s for file %s could not be tagged as 'locked'." % (stringhash, filename))
+ logger.error("Response: %s", buf.getvalue())
+ c.close()
+ continue
+ if c.getinfo(HTTP_CODE) != 200:
+ logger.error("Following string hash %s for file %s could not be tagged as 'locked'" % (stringhash, filename))
+ logger.error("Response: %s", buf.getvalue())
+ c.close()
+ continue
+ c.close()
+
+ if debug:
+ logger.debug('---------------------')
+ logger.debug(buf.getvalue())
+
+if __name__ == '__main__':
+ # python _utils/tag_as_locked.py _utils/tx-resource-names _utils/tx-sourcesnames api-token --debug --manual
+ parser = ArgumentParser()
+ # provide the file from tx configuration containing the resource names
+ parser.add_argument("tx_resourcenamesfile")
+ # provide the file from tx configuration containing only the original source filenames
+ parser.add_argument("tx_sourcesnamesfile")
+ # whether or not to write debug json files
+ parser.add_argument("--debug", action='store_true')
+ # whether or not to tag file by a file
+ parser.add_argument("--manual", action='store_true')
+
+
+ args = parser.parse_args()
+ configure_logging(LOG_FILE_NAME)
+
+ manual = args.manual
+
+ if not isfile(args.tx_resourcenamesfile):
+ print("please check your transifex resource names file")
+ logger.error("please check your transifex resource names file")
+ sys.exit(1)
+
+ if not isfile(args.tx_sourcesnamesfile):
+ print("please check your transifex original sourcenames file")
+ logger.error("please check your transifex original sourcenames file")
+ sys.exit(1)
+
+ tx_resources = []
+ with iopen(args.tx_resourcenamesfile) as f:
+ tx_resources = f.readlines()
+ tx_resources = [ t.rstrip() for t in tx_resources]
+
+
+ # testing on
+ if manual:
+ manual_break()
+
+ if 'TX_TOKEN' not in environ:
+ parser.error('TX_TOKEN variable not set')
+ tx_api_token = environ['TX_TOKEN']
+ perms_and_redirects = get_all_original_permalinks_and_redirects(args.tx_sourcesnamesfile)
+ hash_and_tags_mapping = create_hash_and_tags_mapping(tx_resources, tx_api_token, args.debug, perms_and_redirects, manual)
+
+ if args.debug:
+ logger.debug("------------------------------------------------")
+ logger.debug("----------HASH 2 TAG NOTRANSLATE MAPPING--------")
+ logger.debug("------------------------------------------------")
+ logger.debug(dumps(hash_and_tags_mapping, indent=4))
+ logger.debug("------------------------------------------------")
+ logger.debug("------------------------------------------------")
+ logger.debug("------------------------------------------------")
+ logger.debug("-------------STRINGS TAGGED NOTRANSLATE---------")
+ logger.debug("------------------------------------------------")
+ logger.debug("------------------------------------------------")
+ logger.debug(dumps(tagged_locked, indent=4))
+ logger.debug("------------------------------------------------")
+ logger.debug("------------------------------------------------")
+ logger.debug("------------------------------------------------")
+
+ tag_strings_as_locked(hash_and_tags_mapping, tx_api_token, args.debug)
+
+
+
+
diff --git a/_utils/_translation_utils/tx_config.sh b/_utils/_translation_utils/tx_config.sh
new file mode 100644
index 00000000..b028716b
--- /dev/null
+++ b/_utils/_translation_utils/tx_config.sh
@@ -0,0 +1,49 @@
+#!/bin/bash
+# to be run from the git root
+#to incoporate them back, no need
+
+set -e
+
+# add ref & lang attributes to newly created files
+python3 _utils/_translation_utils/prepare_for_translation.py en _doc/ _utils/_translation_utils/COUNTER.txt
+
+# because there is apparently a feature in tx config that doe snot update an existing configuration, every time everythin will be done from scratch:
+# delete current tx configuration
+
+mv .tx/config /tmp/tx_config_old || :
+
+#init a tx configuration
+tx init --skipsetup
+
+# map the files with tx config
+tx config mapping-bulk -p qubes --source-language en --type GITHUBMARKDOWN -f '.md' -d --source-file-dir _doc -i _dev --expression '_translated//_doc/{filepath}/{filename}{extension}' --execute
+tx config mapping-bulk -p qubes --source-language en --type GITHUBMARKDOWN -f '.md' -d --source-file-dir pages --expression '_translated//pages/{filepath}/{filename}{extension}' --execute
+tx config mapping-bulk -p qubes --source-language en --type GITHUBMARKDOWN -f '.md' -d --source-file-dir news --expression '_translated//news/{filepath}/{filename}{extension}' --execute
+
+#HTML
+tx config mapping -r qubes._doc_introduction_intro --source-lang en --type HTML --source-file _doc/introduction/intro.md '_translated//_doc/introduction/intro.md' --execute
+tx config mapping -r qubes.pages_partners --source-lang en --type HTML --source-file pages/partners.html '_translated//pages/partners.html' --execute
+tx config mapping -r qubes.pages_home --source-lang en --type HTML --source-file pages/home.html '_translated//pages/home.html' --execute
+
+tx config mapping -r qubes.data_architecture --source-lang en --type YAML_GENERIC --source-file _data/architecture.yml --expression '_translated//_data//architecture.yml' --execute
+tx config mapping -r qubes.data_doc_index --source-lang en --type YAML_GENERIC --source-file _data/doc-index.yml --expression '_translated//_data//doc-index.yml' --execute
+tx config mapping -r qubes.data_includes --source-lang en --type YAML_GENERIC --source-file _data/includes.yml --expression '_translated//_data//includes.yml' --execute
+tx config mapping -r qubes.data_team-page --source-lang en --type YAML_GENERIC --source-file _data/team-page.yml --expression '_translated//_data//team-page.yml' --execute
+tx config mapping -r qubes.data_team --source-lang en --type YAML_GENERIC --source-file _data/team.yml --expression '_translated//_data//team.yml' --execute
+tx config mapping -r qubes.data_experts --source-lang en --type YAML_GENERIC --source-file _data/experts.yml --expression '_translated//_data//experts.yml' --execute
+tx config mapping -r qubes.data_downloads_page --source-lang en --type YAML_GENERIC --source-file _data/downloads-page.yml --expression '_translated//_data//downloads-page.yml' --execute
+tx config mapping -r qubes.data_hcl --source-lang en --type YAML_GENERIC --source-file _data/hcl.yml --expression '_translated//_data//hcl.yml' --execute
+tx config mapping -r qubes.data_research --source-lang en --type YAML_GENERIC --source-file _data/research.yml --expression '_translated//_data//research.yml' --execute
+tx config mapping -r qubes.data_style_guide_page --source-lang en --type YAML_GENERIC --source-file _data/style-guide-page.yml --expression '_translated//_data//style-guide-page.yml' --execute
+
+crudini --del .tx/config qubes._doc_README
+crudini --del .tx/config qubes._doc_CONTRIBUTING
+
+sed -i 's/\._doc_/.doc_/' .tx/config
+
+echo "#####################################################################################"
+echo "############# Please pay attention to the changes made to the current tx config #####"
+echo "############# Do you have to delete some resources on transifex manually? ###########"
+echo "############# left is the current, on the right is the old tx config ################"
+echo "#####################################################################################"
+diff /tmp/tx_config_old .tx/config --color || :
diff --git a/_utils/_translation_utils/tx_pull.sh b/_utils/_translation_utils/tx_pull.sh
new file mode 100644
index 00000000..82061583
--- /dev/null
+++ b/_utils/_translation_utils/tx_pull.sh
@@ -0,0 +1,10 @@
+#!/bin/bash -e
+# first argument is the language for which the translated files should be downloaded
+# the mode developer will download all unreviewed translated strings as well
+tx pull --force -l $1 --mode reviewed --traceback -r qubes.doc*
+tx pull --force -l $1 --mode reviewed --traceback -r qubes.pages*
+tx pull --force -l $1 --mode reviewed --traceback -r qubes.news*
+
+# the different mode here is needed for YAML files, since the developer mode does not download source strings if untranslated
+# the mode reviewed here will not work and will return empty strings for nontranslated ones
+tx pull --force -l $1 --mode sourceastranslation --traceback -r qubes.data_*
diff --git a/_utils/transifex-pull b/_utils/transifex-pull
new file mode 100755
index 00000000..282262c6
--- /dev/null
+++ b/_utils/transifex-pull
@@ -0,0 +1,38 @@
+#!/bin/sh
+
+# Pull translated pages from Transifex
+
+if ! command -v tx; then
+ echo "transifex-client needs to be installed" >&2
+ exit 1
+fi
+
+if [ -z "$TX_TOKEN" ]; then
+ echo "Please set TX_TOKEN variable to Transifex API key" >&2
+ exit 1
+fi
+
+if [ -z "$1" ]; then
+ echo "Usage: $0 [ ...]"
+fi
+
+set -e
+
+for lang in "$@"; do
+ bash _utils/_translation_utils/tx_pull.sh "$lang"
+ bash _utils/_translation_utils/post_transifex_pull.sh "$lang" _translated/"$lang"
+done
+
+bash _utils/_translation_utils/check_all_langs.sh _translated
+
+# switch to ssh for push
+git -C _translated remote set-url origin git@github.com:QubesOS/qubes-translated
+
+# commit and push
+git -C _translated add .
+
+# if nothing to commit, exit early
+[ -n "$(git -C _translated status --porcelain)" ] || exit 0
+
+git -C _translated commit -m 'Update translated content'
+git -C _translated push origin HEAD:master
diff --git a/_utils/transifex-push b/_utils/transifex-push
new file mode 100755
index 00000000..7d17980d
--- /dev/null
+++ b/_utils/transifex-push
@@ -0,0 +1,27 @@
+#!/bin/sh
+
+# This script uploads the website source files to Transifex for translation, including all the preparatory work
+
+if ! command -v tx; then
+ echo "transifex-client needs to be installed" >&2
+ exit 1
+fi
+
+if [ -z "$TX_TOKEN" ]; then
+ echo "Please set TX_TOKEN variable to Transifex API key" >&2
+ exit 1
+fi
+
+set -e
+
+# update config
+bash _utils/_translation_utils/tx_config.sh
+
+# push the sources
+tx push -s
+
+# update sources metadata (notranslate tags, locks etc)
+bash _utils/_translation_utils/prepare_tx_config_for_notranslate_tags.sh .tx/config /tmp/tx-resources-names.txt /tmp/tx-sources-filenames.txt
+python3 _utils/_translation_utils/tag_strings_as_locked.py /tmp/tx-resources-names.txt /tmp/tx-sources-filenames.txt --debug
+
+
diff --git a/_utils/update-translated b/_utils/update-translated
new file mode 100755
index 00000000..9164359c
--- /dev/null
+++ b/_utils/update-translated
@@ -0,0 +1,24 @@
+#!/bin/bash
+
+set -e
+
+tmpbranch="new-$$"
+git -C _translated fetch origin master:"$tmpbranch"
+tmpdir=$(mktemp -d)
+trap 'rm -rf $tmpdir; git -C _translated br -D $tmpbranch' EXIT
+git clone --shared "$PWD/_translated" -b "$tmpbranch" "$tmpdir/translated"
+
+if ! _utils/verify-translated "$tmpdir/translated"; then
+ echo "Translated content did not pass sanity check, not updating" >&2
+ # TODO: consider some louder alert? email? issue on github?
+ exit 1
+fi
+
+git -C _translated merge --ff-only "$tmpbranch"
+git add _translated
+
+git commit -m 'autoupdate: _translated'
+commit_id=$(git show --pretty=format:%H|head -1)
+tag_name=auto_${commit_id:0:8}
+git tag -s -m "Automatic tag for commit $commit_id" "$tag_name"
+git push origin master $tag_name
diff --git a/_utils/verify-translated b/_utils/verify-translated
new file mode 100755
index 00000000..0bfbc0a2
--- /dev/null
+++ b/_utils/verify-translated
@@ -0,0 +1,201 @@
+#!/usr/bin/env python3
+
+# Simple verifier for qubes-translated repository content sanity.
+# This script looks at frontmatter of each file mostly checks that:
+# - lang: key is present and matches the directory name
+# - ref: key is present
+# - url-related parameters (redirect_from, permalink) are properly language-scoped
+# - no unexpected entries in the frontmatter are present
+#
+# Besides that, verifies if no unexpected files are present.
+#
+# Usage:
+# $0
+# Supported env variables:
+# - TRANSLATED_LANGS - list of expected languages
+
+import argparse
+import os
+import string
+import yaml
+
+TRANSLATED_LANGS = ['de']
+if 'TRANSLATED_LANGS' in os.environ:
+ TRANSLATED_LANGS = os.environ['TRANSLATED_LANGS'].split()
+
+SAFE_PATH_CHARS = string.ascii_letters + string.digits + '/_-.'
+
+ALLOWED_FRONTMATTER_KEYS = ['title', 'lang', 'ref', 'permalink', 'redirect_from', 'redirect_to', 'layout', 'model']
+
+MANDATORY_FRONTMATTER_KEYS = ['title', 'lang', 'ref', 'layout']
+
+ALLOWED_EXTERNAL_REDIRECT_TARGETS = ['https://github.com/Qubes-Community/Contents/blob/master/']
+
+ALLOWED_LAYOUTS = ['doc', 'doc-index', 'doc-full', 'hcl', 'downloads', 'news', 'team', 'home', 'default', 'experts', 'sidebar']
+
+#TODO: consider allowing some layouts only on some pages (and consider their redirect_from too)
+
+parser = argparse.ArgumentParser()
+parser.add_argument('directory')
+
+class VerificationError(Exception):
+ def __init__(self, path, msg):
+ safe_name = ''.join(l if l in SAFE_PATH_CHARS else '?' for l in path)
+ super().__init__('{}: {}'.format(path, msg))
+
+def verify_readme(path):
+ with open(path) as f:
+ readme_text = f.read()
+
+ if '---' in readme_text:
+ raise VerificationError(path, 'may not contain frontmatter')
+ if '<' in readme_text:
+ raise VerificationError(path, 'may not contain HTML')
+ if '{' in readme_text:
+ raise VerificationError(path, 'may not contain liquid templates')
+
+
+def verify_md_file(lang, path):
+ with open(path) as f:
+ file_content = f.read()
+
+ # there must be frontmatter
+ if not file_content.startswith('---\n'):
+ raise VerificationError(path, 'missing frontmatter')
+
+ # better be more strict - may catch too much (if another separator is
+ # used), but then loading yaml will detect multiple documents
+ frontmatter_text = file_content[4:].split('\n---\n')[0]
+ # there could be _just_ frontmatter too
+ if frontmatter_text.endswith('\n---'):
+ frontmatter_text = frontmatter_text[:-4]
+ try:
+ frontmatter = yaml.safe_load(frontmatter_text)
+ except Exception as e:
+ raise VerificationError(path, 'failed to parse frontmatter: {!s}'.format(e)) from e
+ verify_frontmatter(lang, path, frontmatter)
+
+
+def verify_frontmatter(lang, path, frontmatter):
+ # double check if all entries were verified
+ verified = []
+
+ url_prefix = '/{}/'.format(lang)
+ if any(key not in ALLOWED_FRONTMATTER_KEYS for key in frontmatter):
+ raise VerificationError(path, 'unexpected frontmatter key')
+
+ for key in MANDATORY_FRONTMATTER_KEYS:
+ if key not in frontmatter:
+ raise VerificationError(path, key + ' missing in frontmatter')
+
+ if lang != frontmatter['lang']:
+ raise VerificationError(path, 'lang mismatch')
+
+ verified.append('lang')
+
+ if not isinstance(frontmatter['ref'], int):
+ raise VerificationError(path, 'invalid ref format')
+
+ verified.append('ref')
+
+ if 'permalink' in frontmatter:
+ if not frontmatter['permalink'].startswith(url_prefix):
+ raise VerificationError(path, 'invalid permalink')
+
+ verified.append('permalink')
+
+ if 'redirect_from' in frontmatter:
+ if isinstance(frontmatter['redirect_from'], str):
+ redirect_from = [frontmatter['redirect_from']]
+ elif isinstance(frontmatter['redirect_from'], list):
+ redirect_from = frontmatter['redirect_from']
+ else:
+ raise VerificationError(path, 'invalid redirect_from format')
+
+ for url in redirect_from:
+ if '/..' in url:
+ raise VerificationError(path, '.. in url')
+ if not url.startswith(url_prefix):
+ raise VerificationError(path, 'invalid redirect_from')
+
+ verified.append('redirect_from')
+
+ if 'redirect_to' in frontmatter:
+ url = frontmatter['redirect_to']
+ if isinstance(url, list):
+ if len(url) != 1:
+ raise VerificationError(path, 'if redirect_to is a list, must be 1-element')
+ url = url[0]
+ if not isinstance(url, str):
+ raise VerificationError(path, 'invalid redirect_to format')
+ if '/..' in url:
+ raise VerificationError(path, '.. in redirect_to')
+ if not any(url.startswith(prefix) for prefix in ALLOWED_EXTERNAL_REDIRECT_TARGETS + [url_prefix]):
+ raise VerificationError(path, 'forbidden redirect_to target')
+
+ verified.append('redirect_to')
+
+ if frontmatter['layout'] not in ALLOWED_LAYOUTS:
+ raise VerificationError(path, 'forbidden layout')
+
+ verified.append('layout')
+
+ title = frontmatter['title']
+ if not isinstance(title, str):
+ raise VerificationError(path, 'invalid title format')
+
+ # avoid HTML in title
+ if '<' in title or '%' in title:
+ raise VerificationError(path, 'invalid character in title')
+
+ verified.append('title')
+
+ # if 'model' is present, must have 'all' value
+ if 'model' in frontmatter:
+ if frontmatter['model'] != 'all':
+ raise VerificationError(path, 'invalid model value')
+ verified.append('model')
+
+ # intentionally compare lists, not sets, to catch duplicates too
+ if sorted(verified) != sorted(frontmatter):
+ raise VerificationError(path, 'BUG, some frontmatter entries were not verified')
+
+ # all is ok
+
+
+def verify_lang(lang, path):
+ for dirpath, dirnames, filenames in os.walk(path):
+ for filename in filenames:
+ filepath = os.path.join(dirpath, filename)
+ if any(c not in SAFE_PATH_CHARS for c in filename):
+ raise VerificationError(filepath, 'unsafe characters in filename')
+
+ if filename.endswith('.md'):
+ verify_md_file(lang, filepath)
+ elif filename.endswith('.html'):
+ # the frontmatter is expected the same
+ verify_md_file(lang, filepath)
+ elif filename.endswith('.yml'):
+ # those are loaded scoped anyway, so can mess only own language
+ pass
+ else:
+ raise VerificationError(filepath, 'unexpected file type')
+
+
+
+def main():
+ args = parser.parse_args()
+
+ for lang in os.listdir(args.directory):
+ if lang == '.git':
+ pass
+ elif lang == 'README.md':
+ verify_readme(os.path.join(args.directory, lang))
+ elif lang in TRANSLATED_LANGS:
+ verify_lang(lang, os.path.join(args.directory, lang))
+ else:
+ raise VerificationError(lang, 'unexpected language dir')
+
+
+if __name__ == '__main__':
+ main()
Qubes OS