From 0cca3991a0ffc86d79418464bc43abefc8bfd73d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marek=20Marczykowski-G=C3=B3recki?= Date: Thu, 24 Jun 2021 05:00:01 +0200 Subject: [PATCH] translation: Move site verification after downloading all languages Do not attempt to verify correctness of the website before refreshing all languages. If any structural change happened (rename, permalink change etc), it will fail because of the old files, not the freshly downloaded ones. Make htmlproofer postprocessing script language agnostic. This is mostly about removing "language" parameter, which wasn't used anyway. --- _utils/_translation_utils/check_all_langs.sh | 28 +++++++++++++++++++ .../_translation_utils/post_transifex_pull.sh | 23 --------------- .../postprocess_htmlproofer.py | 21 ++++---------- _utils/transifex-pull | 2 ++ 4 files changed, 36 insertions(+), 38 deletions(-) create mode 100644 _utils/_translation_utils/check_all_langs.sh diff --git a/_utils/_translation_utils/check_all_langs.sh b/_utils/_translation_utils/check_all_langs.sh new file mode 100644 index 000000000..3755042c8 --- /dev/null +++ b/_utils/_translation_utils/check_all_langs.sh @@ -0,0 +1,28 @@ +#!/bin/bash +# to be run from the git root +# $1 is directory where translated files reside and language needs to be added to internal urls +# TODO param check + +set -e + +echo "================================= build site ==================================" +#read b +bundle exec jekyll b + +all_ok=true +echo "================================= run htmlproofer ===============================" +htmlproofer ./_site --disable-external --checks-to-ignore ImageCheck --file-ignore "./_site/video-tours/index.html,./_site/.*/video-tours/index.html" --url-ignore "/qubes-issues/" --log-level debug 2&> /tmp/html.output || all_ok=false + +# exit here if all is ok +if $all_ok; then + echo 'All checks passed!' + exit +fi + +echo "================================== as a last resort in case of errors process html proofer errors =================================" +python3 _utils/_translation_utils/postprocess_htmlproofer.py /tmp/html.output "$1" + +echo "================================= build the site and run htmlproofer ====================================" +rm -rf ./_site/ +bundle exec jekyll b +htmlproofer ./_site --disable-external --checks-to-ignore ImageCheck --file-ignore "./_site/video-tours/index.html,./_site/.*/video-tours/index.html" --url-ignore "/qubes-issues/" --log-level debug diff --git a/_utils/_translation_utils/post_transifex_pull.sh b/_utils/_translation_utils/post_transifex_pull.sh index fc8cd2d90..9e6f6f563 100644 --- a/_utils/_translation_utils/post_transifex_pull.sh +++ b/_utils/_translation_utils/post_transifex_pull.sh @@ -25,26 +25,3 @@ python3 _utils/_translation_utils/postprocess_translation.py "$1" "$2" /tmp/tx- echo "============================ post processing step 4 press to cont ======================================" #read b bash _utils/_translation_utils/postprocess_translation.sh "$1" "$2" /tmp/translated_href_urls.txt - - -echo "================================= build site ==================================" -#read b -bundle exec jekyll b - -all_ok=true -echo "================================= run htmlproofer ===============================" -htmlproofer ./_site --disable-external --checks-to-ignore ImageCheck --file-ignore "./_site/video-tours/index.html,./_site/$1/video-tours/index.html" --url-ignore "/qubes-issues/" --log-level debug 2&> /tmp/html.output || all_ok=false - -# exit here if all is ok -if $all_ok; then - echo 'All checks passed!' - exit -fi - -echo "================================== as a last resort in case of errors process html proofer errors =================================" -python3 _utils/_translation_utils/postprocess_htmlproofer.py "$1" /tmp/html.output "$2" - -echo "================================= build the site and run htmlproofer ====================================" -rm -rf ./_site/ -bundle exec jekyll b -htmlproofer ./_site --disable-external --checks-to-ignore ImageCheck --file-ignore "./_site/video-tours/index.html,./_site/$1/video-tours/index.html" --url-ignore "/qubes-issues/" --log-level debug diff --git a/_utils/_translation_utils/postprocess_htmlproofer.py b/_utils/_translation_utils/postprocess_htmlproofer.py index eb6a587b2..e73d98d95 100644 --- a/_utils/_translation_utils/postprocess_htmlproofer.py +++ b/_utils/_translation_utils/postprocess_htmlproofer.py @@ -1,8 +1,7 @@ #!/usr/bin/python3 ''' -python _utils/_translation_utils/postprocess_htmlproofer.py +python _utils/_translation_utils/postprocess_htmlproofer.py invoke: python _utils/_translation_utils/postprocess_htmlproofer.py de /tmp/html.output _translated/de/ -[de]: translation language [/tmp/html.output]: output from htmlproofer [_translated/de/]: the directory with the downloaded translated files from transifex ''' @@ -123,12 +122,11 @@ def process_markdown(translated_file, internal_links): -def get_all_translated_permalinks_and_redirects_to_file_mapping(translated_dir, lang): +def get_all_translated_permalinks_and_redirects_to_file_mapping(translated_dir): """ - traverse the already updated (via tx pull) root directory with all the translated files for a specific language - and get their permalinks and redirects without the specific language - translated_dir: root directory with all the translated files for a specific language - lang: the specific language + traverse the already updated (via tx pull) root directory with all the translated files + and get their permalinks and redirects + translated_dir: root directory with all the translated files return: set holding the translated permalinks and redirects """ mapping = {} @@ -246,8 +244,6 @@ def process_yml(translated, errorlinks): if __name__ == '__main__': # python _utils/_translation_utils/postprocess_htmlproofer.py de /tmp/html.output _translated/de/ parser = ArgumentParser() - # for which language should we do this - parser.add_argument("language") # the file containing the output of htmlproofer parser.add_argument("htmlproofer_output") # the directory containing the translated (downloaded via tx pull) files @@ -262,11 +258,6 @@ def process_yml(translated, errorlinks): logger.error("please check your translated directory") exit(1) - if not args.language in TRANSLATED_LANGS: - print("language not in the expected translation languages") - logger.error("please check your translation language") - exit(1) - if not isfile(args.htmlproofer_output): print("please check your html proofer output file") logger.error("please check your html proofer output file") @@ -293,7 +284,7 @@ def process_yml(translated, errorlinks): logger.debug("------------------------------------------------") logger.debug("------------------------------------------------") - mapping, yml_files = get_all_translated_permalinks_and_redirects_to_file_mapping(args.translated_dir, args.language) + mapping, yml_files = get_all_translated_permalinks_and_redirects_to_file_mapping(args.translated_dir) log_debug('mapping ', mapping) diff --git a/_utils/transifex-pull b/_utils/transifex-pull index 6c149cb10..4a83fd6f6 100755 --- a/_utils/transifex-pull +++ b/_utils/transifex-pull @@ -23,6 +23,8 @@ for lang in "$@"; do bash _utils/_translation_utils/post_transifex_pull.sh "$lang" _translated/"$lang" done +bash _utils/_translation_utils/check_all_langs.sh + # switch to ssh for push git -C _translated remote set-url origin git@github.com:QubesOS/qubes-translated