From c0b2f6ea501f8947b1e9e645377bdb80cf0b8975 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C3=81lvaro=20Mond=C3=A9jar?= Date: Sun, 27 Feb 2022 00:31:02 +0100 Subject: [PATCH] Add config setting to disable cross language search (#32) * Add config setting to disable cross language search * Bump version * Continue work... * Add support for readthedocs theme * Fix test * Update docs * Fix error removing suffixes * Add tests for search indexes * Test config * Improve documentation for the new config setting --- .bumpversion.cfg | 2 +- docs/locale/es/config.md.po | 18 + docs/src/config.md | 10 + examples/material/mkdocs.yml | 3 +- .../mkdocs-theme/docs/locale/es/index.md.po | 12 + .../mkdocs-theme/docs/locale/fr/index.md.po | 12 + examples/mkdocs-theme/docs/src/index.md | 3 + examples/mkdocs-theme/mkdocs.yml | 16 + examples/po-outside-docs/mkdocs.yml | 1 + .../docs/locale/es/index.md.po | 12 + .../docs/locale/fr/index.md.po | 12 + examples/readthedocs-theme/docs/src/index.md | 3 + examples/readthedocs-theme/mkdocs.yml | 19 + mkdocs.yml | 1 + mkdocs_mdpo_plugin/__init__.py | 2 +- mkdocs_mdpo_plugin/compat.py | 4 + mkdocs_mdpo_plugin/config.py | 9 + mkdocs_mdpo_plugin/plugin.py | 41 +- mkdocs_mdpo_plugin/search_indexes.py | 367 ++++++++++++++++++ mkdocs_mdpo_plugin/translations.py | 9 +- setup.cfg | 2 +- tests/test_examples.py | 34 ++ tests/test_plugin_config.py | 12 + tests/test_search_indexes.py | 200 ++++++++++ 24 files changed, 794 insertions(+), 10 deletions(-) create mode 100644 examples/mkdocs-theme/docs/locale/es/index.md.po create mode 100644 examples/mkdocs-theme/docs/locale/fr/index.md.po create mode 100644 examples/mkdocs-theme/docs/src/index.md create mode 100644 examples/mkdocs-theme/mkdocs.yml create mode 100644 examples/readthedocs-theme/docs/locale/es/index.md.po create mode 100644 examples/readthedocs-theme/docs/locale/fr/index.md.po create mode 100644 examples/readthedocs-theme/docs/src/index.md create mode 100644 examples/readthedocs-theme/mkdocs.yml create mode 100644 mkdocs_mdpo_plugin/compat.py create mode 100644 mkdocs_mdpo_plugin/search_indexes.py create mode 100644 tests/test_examples.py create mode 100644 tests/test_search_indexes.py diff --git a/.bumpversion.cfg b/.bumpversion.cfg index 68b2033..41ffefb 100644 --- a/.bumpversion.cfg +++ b/.bumpversion.cfg @@ -1,5 +1,5 @@ [bumpversion] -current_version = 0.0.20 +current_version = 0.0.21 [bumpversion:file:mkdocs_mdpo_plugin/__init__.py] diff --git a/docs/locale/es/config.md.po b/docs/locale/es/config.md.po index 74abffb..87defa6 100644 --- a/docs/locale/es/config.md.po +++ b/docs/locale/es/config.md.po @@ -188,3 +188,21 @@ msgstr "" msgid "Content" msgstr "Contenido" + +msgid "" +"It configures if the search plugin of the theme will search through all " +"languages. By default is enabled. You can disable it to restrict the search " +"to the active language." +msgstr "" +"Configura si el plugin de búsqueda del tema buscará a través de todos los " +"idiomas. Por defecto está habilitado. Puedes deshabilitarlo para restringir " +"la búsqueda al idioma activo." + +msgid "" +"The support for this feature currently includes the [mkdocs-material] theme," +" the Mkdocs theme, the Readthedocs theme and all themes which use the " +"builtin Mkdocs search plugin." +msgstr "" +"El soporte de esta característica actualmente incluye el tema [mkdocs-" +"material], el tema por defecto de Mkdocs, el tema de Readthedocs y todos los " +"temas que usen el plugin de búsqueda incluido en Mkdocs." diff --git a/docs/src/config.md b/docs/src/config.md index 826b661..4e08d09 100644 --- a/docs/src/config.md +++ b/docs/src/config.md @@ -203,6 +203,16 @@ File extensions that are ignored from being added to site directory, defaults to You can ignore certain messages from being dumped into PO files adding them to this list. + +### **`cross_language_search`** (*bool*) + +It configures if the search plugin of the theme will search through all +languages. By default is enabled. You can disable it to restrict the search to the active language. + +The support for this feature currently includes the [mkdocs-material] theme, +the Mkdocs theme, the Readthedocs theme and all themes which use the builtin +Mkdocs search plugin. + [iso-369]: https://en.wikipedia.org/wiki/ISO_639 [mkdocs-material]: https://squidfunk.github.io/mkdocs-material [mkdocs-material-site-language]: https://squidfunk.github.io/mkdocs-material/setup/changing-the-language/#site-language diff --git a/examples/material/mkdocs.yml b/examples/material/mkdocs.yml index 8a28086..44b8fd4 100644 --- a/examples/material/mkdocs.yml +++ b/examples/material/mkdocs.yml @@ -11,7 +11,8 @@ nav: plugins: - search - - mdpo + - mdpo: + cross_language_search: false extra: alternate: diff --git a/examples/mkdocs-theme/docs/locale/es/index.md.po b/examples/mkdocs-theme/docs/locale/es/index.md.po new file mode 100644 index 0000000..84f051b --- /dev/null +++ b/examples/mkdocs-theme/docs/locale/es/index.md.po @@ -0,0 +1,12 @@ +# +msgid "" +msgstr "" + +msgid "Home" +msgstr "Inicio" + +msgid "Welcome to MkDocs" +msgstr "Bienvenido a Mkdocs" + +msgid "Some content" +msgstr "Algo de contenido" diff --git a/examples/mkdocs-theme/docs/locale/fr/index.md.po b/examples/mkdocs-theme/docs/locale/fr/index.md.po new file mode 100644 index 0000000..b31bec7 --- /dev/null +++ b/examples/mkdocs-theme/docs/locale/fr/index.md.po @@ -0,0 +1,12 @@ +# +msgid "" +msgstr "" + +msgid "Home" +msgstr "Accueil" + +msgid "Welcome to MkDocs" +msgstr "Bienvenue sur mkdocs" + +msgid "Some content" +msgstr "Du contenu" diff --git a/examples/mkdocs-theme/docs/src/index.md b/examples/mkdocs-theme/docs/src/index.md new file mode 100644 index 0000000..6bc8680 --- /dev/null +++ b/examples/mkdocs-theme/docs/src/index.md @@ -0,0 +1,3 @@ +# Welcome to MkDocs + +Some content diff --git a/examples/mkdocs-theme/mkdocs.yml b/examples/mkdocs-theme/mkdocs.yml new file mode 100644 index 0000000..3ae9ffe --- /dev/null +++ b/examples/mkdocs-theme/mkdocs.yml @@ -0,0 +1,16 @@ +site_name: mkdocs-mdpo-plugin Mkdocs theme example +site_url: https://mkdocs-mdpo.ga +docs_dir: docs/src + +nav: + - Home: index.md + +plugins: + - search + - mdpo: + languages: + - en + - es + - fr + cross_language_search: false + locale_dir: ../locale diff --git a/examples/po-outside-docs/mkdocs.yml b/examples/po-outside-docs/mkdocs.yml index 1e54591..72681f8 100644 --- a/examples/po-outside-docs/mkdocs.yml +++ b/examples/po-outside-docs/mkdocs.yml @@ -13,6 +13,7 @@ plugins: - search - mdpo: locale_dir: ../locale + cross_language_search: false extra: alternate: diff --git a/examples/readthedocs-theme/docs/locale/es/index.md.po b/examples/readthedocs-theme/docs/locale/es/index.md.po new file mode 100644 index 0000000..84f051b --- /dev/null +++ b/examples/readthedocs-theme/docs/locale/es/index.md.po @@ -0,0 +1,12 @@ +# +msgid "" +msgstr "" + +msgid "Home" +msgstr "Inicio" + +msgid "Welcome to MkDocs" +msgstr "Bienvenido a Mkdocs" + +msgid "Some content" +msgstr "Algo de contenido" diff --git a/examples/readthedocs-theme/docs/locale/fr/index.md.po b/examples/readthedocs-theme/docs/locale/fr/index.md.po new file mode 100644 index 0000000..b31bec7 --- /dev/null +++ b/examples/readthedocs-theme/docs/locale/fr/index.md.po @@ -0,0 +1,12 @@ +# +msgid "" +msgstr "" + +msgid "Home" +msgstr "Accueil" + +msgid "Welcome to MkDocs" +msgstr "Bienvenue sur mkdocs" + +msgid "Some content" +msgstr "Du contenu" diff --git a/examples/readthedocs-theme/docs/src/index.md b/examples/readthedocs-theme/docs/src/index.md new file mode 100644 index 0000000..6bc8680 --- /dev/null +++ b/examples/readthedocs-theme/docs/src/index.md @@ -0,0 +1,3 @@ +# Welcome to MkDocs + +Some content diff --git a/examples/readthedocs-theme/mkdocs.yml b/examples/readthedocs-theme/mkdocs.yml new file mode 100644 index 0000000..fcf6eb9 --- /dev/null +++ b/examples/readthedocs-theme/mkdocs.yml @@ -0,0 +1,19 @@ +site_name: mkdocs-mdpo-plugin Mkdocs theme example +site_url: https://mkdocs-mdpo.ga +docs_dir: docs/src + +nav: + - Home: index.md + +theme: + name: readthedocs + +plugins: + - search + - mdpo: + languages: + - en + - es + - fr + cross_language_search: false + locale_dir: ../locale diff --git a/mkdocs.yml b/mkdocs.yml index 2b20ffb..252b3a5 100644 --- a/mkdocs.yml +++ b/mkdocs.yml @@ -81,6 +81,7 @@ plugins: root_domain: true - mdpo: locale_dir: ../locale + cross_language_search: false - minify: minify_html: true diff --git a/mkdocs_mdpo_plugin/__init__.py b/mkdocs_mdpo_plugin/__init__.py index a4f7273..46be9c2 100644 --- a/mkdocs_mdpo_plugin/__init__.py +++ b/mkdocs_mdpo_plugin/__init__.py @@ -1,3 +1,3 @@ """mkdocs-mdpo-plugin package""" -__version__ = '0.0.20' +__version__ = '0.0.21' diff --git a/mkdocs_mdpo_plugin/compat.py b/mkdocs_mdpo_plugin/compat.py new file mode 100644 index 0000000..6bfd20f --- /dev/null +++ b/mkdocs_mdpo_plugin/compat.py @@ -0,0 +1,4 @@ +def removesuffix(s, suf): + if suf and s.endswith(suf): + return s[:-len(suf)] + return s diff --git a/mkdocs_mdpo_plugin/config.py b/mkdocs_mdpo_plugin/config.py index 6f20684..3ff2519 100644 --- a/mkdocs_mdpo_plugin/config.py +++ b/mkdocs_mdpo_plugin/config.py @@ -19,6 +19,7 @@ ), ('ignore_extensions', Type(list, default=['.po', '.pot', '.mo'])), ('ignore_msgids', Type(list, default=[])), + ('cross_language_search', Type(bool, default=True)), ) @@ -146,6 +147,14 @@ def _languages_required(): ) config['theme'].dirs.insert(0, custom_sitemap_dir) + # check that cross language search configuration is valid + if plugin.config.get('cross_language_search') is False: + if 'search' not in config['plugins']: + raise ValidationError( + '"cross_language_search" setting is disabled but' + ' no "search" plugin has been added to "plugins"', + ) + # store reference in plugin to markdown_extensions for later usage plugin.extensions.markdown = markdown_extensions diff --git a/mkdocs_mdpo_plugin/plugin.py b/mkdocs_mdpo_plugin/plugin.py index 2e776ef..f89ed05 100644 --- a/mkdocs_mdpo_plugin/plugin.py +++ b/mkdocs_mdpo_plugin/plugin.py @@ -11,6 +11,7 @@ from mdpo.md2po import Md2Po from mdpo.po2md import Po2Md +from mkdocs_mdpo_plugin.compat import removesuffix from mkdocs_mdpo_plugin.config import CONFIG_SCHEME, on_config_event from mkdocs_mdpo_plugin.extensions import Extensions from mkdocs_mdpo_plugin.mdpo_events import ( @@ -24,6 +25,7 @@ MkdocsBuild, set_on_build_error_event, ) +from mkdocs_mdpo_plugin.search_indexes import TranslationsSearchPatcher from mkdocs_mdpo_plugin.translations import Translation, Translations @@ -97,7 +99,7 @@ def on_files(self, files, config): dest_path = Template( self.config['dest_filename_template'], ).render(**context) - src_path = f"{dest_path.rstrip('.html')}.md" + src_path = f"{removesuffix(dest_path, '.html')}.md" self.translations.files[file.src_path][language] = ( os.path.join( @@ -329,6 +331,7 @@ def on_page_markdown(self, markdown, page, config, files): _translated_entries_msgstrs.append(entry.msgstr) _translated_entries_msgids.append(entry.msgid) + # create translation object translation = Translation( language, po, @@ -341,14 +344,15 @@ def on_page_markdown(self, markdown, page, config, files): self.translations.current = translation # change file url - url = new_page.file.url.rstrip('.md') + '.html' + url = removesuffix(new_page.file.url, '.md') + '.html' if config['use_directory_urls']: - url = url.rstrip('index.html') + url = removesuffix(url, 'index.html') new_page.file.url = url self.translations.nav[page.title][language] = [ translated_page_title, new_page.file.url, ] + mkdocs.commands.build._populate_page( new_page, config, @@ -391,6 +395,13 @@ def on_post_page(self, output, page, config): elif not render_path.endswith('.html'): render_path += '.html' + # save locations of records with languages for search indexes usage + location = os.path.relpath( + removesuffix(render_path, 'index.html'), + config['site_dir'], + ) + '/' + self.translations.locations[location] = page.file._mdpo_language + with open(render_path, 'w') as f: f.write(output) return output @@ -398,6 +409,26 @@ def on_post_page(self, output, page, config): def on_post_build(self, config): self.translations.tempdir.cleanup() + if not self.config['cross_language_search']: + # cross language search is disabled, so build indexes + # for each language and patch the 'site_dir' directory + search_patcher = TranslationsSearchPatcher( + config['site_dir'], + self.config['languages'], + self.config['default_language'], + # use mkdocs 'search' plugin if the theme + # has not its own implementation + ( + config['theme'].name + if config['theme'].name + in TranslationsSearchPatcher.supported_themes + else 'mkdocs' + ), + self.translations.locations, + ) + search_patcher.patch_site_dir() + + # save PO files for translations in self.translations.all.values(): for translation in translations: translation.po.save(translation.po_filepath) @@ -468,14 +499,14 @@ def on_post_build(self, config): # reset mkdocs build instance MkdocsBuild._instance = None - def on_serve(self, server, builder, **kwargs): + def on_serve(self, *args, **kwargs): # pragma: no cover """When serving with livereload server, prevent a infinite loop if the user edits a PO file if is placed inside documentation directory. """ if '..' not in self.config['locale_dir']: sys.stderr.write( - 'ERROR - ' + 'ERROR [mdpo] - ' "You need to set 'locale_dir' configuration setting" ' pointing to a directory placed outside' " the documentation directory ('docs_dir') in order to" diff --git a/mkdocs_mdpo_plugin/search_indexes.py b/mkdocs_mdpo_plugin/search_indexes.py new file mode 100644 index 0000000..fa4ffef --- /dev/null +++ b/mkdocs_mdpo_plugin/search_indexes.py @@ -0,0 +1,367 @@ +"""Translation search indexes used to restrict the search index to +the current language for each page in site directory. + +The patch implies several steps, which depend on the active theme: + +1. Get 'search_index.json' or equivalent file path. Ussually located + at 'search/search_index.json'. +2. Separate original search index which includes all the records + for all the languages in different search indexes, one per language. + The files are named 'search_index_es.json', 'search_index_fr.json'... +3. Patch the JS files which loads the 'search_index.json' file creating + one for each language. This depends completely on the active theme. +4. Patch HTML files to load these language versions of JS files instead + of the original ones. +5. Additionally, for themes like readthedocs, patchs HTML search files. +""" + +import copy +import json +import os + +from mkdocs_mdpo_plugin.compat import removesuffix + + +def _language_extension_path(path, extension, language, separator='_'): + return f'{path[:-len(extension)]}{separator}{language}{extension}' + +## +# Get worker Javascript files. +## + + +def _material_get_worker_js_files(site_dir): + worker_js_filepath, worker_js_content = None, None + javascripts_dir = os.path.join(site_dir, 'assets', 'javascripts') + for fname in os.listdir(javascripts_dir): + if fname.endswith('.js'): + fpath = os.path.join(javascripts_dir, fname) + with open(fpath) as f: + content = f.read() + if '/search/search_index.json' in content: + worker_js_filepath = fpath + worker_js_content = content + continue + return [{'path': worker_js_filepath, 'content': worker_js_content}] + + +def _mkdocs_get_worker_js_files(site_dir): + worker_js_filepath = os.path.join(site_dir, 'search', 'worker.js') + with open(worker_js_filepath) as f: + worker_js_content = f.read() + + main_js_filepath = os.path.join(site_dir, 'search', 'main.js') + with open(main_js_filepath) as f: + main_js_content = f.read() + + return [ + {'path': worker_js_filepath, 'content': worker_js_content}, + {'path': main_js_filepath, 'content': main_js_content}, + ] + + +THEME_WORKER_FILES_FUNCS = { + 'material': _material_get_worker_js_files, + 'mkdocs': _mkdocs_get_worker_js_files, + 'readthedocs': _mkdocs_get_worker_js_files, +} + +## +# Patch Javascript worker files. +## + + +def _material_patch_worker_js_files(files, language): + worker_js = files[0] + new_path = _language_extension_path(worker_js['path'], '.js', language) + + new_content = worker_js['content'].replace( + '/search/search_index.json', + f'/search/search_index_{language}.json', + ) + with open(new_path, 'w') as f: + f.write(new_content) + + +def _mkdocs_patch_worker_js_files(files, language): + worker_js, main_js = files + + new_worker_js_path = _language_extension_path( + worker_js['path'], + '.js', + language, + ) + new_worker_js_content = worker_js['content'].replace( + 'search_index.json', + f'search_index_{language}.json', + ) + with open(new_worker_js_path, 'w') as f: + f.write(new_worker_js_content) + + new_main_js_path = _language_extension_path( + main_js['path'], + '.js', + language, + ) + new_main_js_content = main_js['content'].replace( + 'worker.js', + f'worker_{language}.js', + ) + with open(new_main_js_path, 'w') as f: + f.write(new_main_js_content) + + +THEME_WORKER_PATCHS_FUNCS = { + 'material': _material_patch_worker_js_files, + 'mkdocs': _mkdocs_patch_worker_js_files, + 'readthedocs': _mkdocs_patch_worker_js_files, +} + +## +# Patch HTML files to load language JS worker files. +## + + +def _material_patch_html_file(fpath, language, worker_files): + with open(fpath) as f: + content = f.read() + + worker_js_fname = os.path.basename(worker_files[0]['path']) + worker_js_fname_lang = _language_extension_path( + worker_js_fname, + '.js', + language, + ) + + new_content = content.replace( + f'assets/javascripts/{worker_js_fname}"', + f'assets/javascripts/{worker_js_fname_lang}"', + ) + with open(fpath, 'w') as f: + f.write(new_content) + + +def _mkdocs_patch_html_file(fpath, language, *args): + with open(fpath) as f: + content = f.read() + + new_content = content.replace( + 'search/main.js', + f'search/main_{language}.js', + ) + with open(fpath, 'w') as f: + f.write(new_content) + + +THEME_HTML_PATCHS_FUNCS = { + 'material': _material_patch_html_file, + 'mkdocs': _mkdocs_patch_html_file, + 'readthedocs': _mkdocs_patch_html_file, +} + +## +# Get search files +## + + +def _readthedocs_get_search_files(site_dir): + search_file_path = os.path.join(site_dir, 'search.html') + with open(search_file_path) as f: + search_file_content = f.read() + return [{'path': search_file_path, 'content': search_file_content}] + + +THEME_GET_SEARCH_FILES_FUNCS = { + 'readthedocs': _readthedocs_get_search_files, +} + +## +# Patch search files +## + + +def _reathedocs_patch_search_files( + fpath, + language, + default_language, + search_files, +): + search_file = search_files[0] + + # create search file for the language if not exists + lang_search_path = _language_extension_path( + search_file['path'], + '.html', + language, + ) + lang_search_path_content = search_file['content'].replace( + 'search.html', + f'search_{language}.html', + ).replace( + f'search/main_{default_language}.js', + f'search/main_{language}.js', + ) + with open(lang_search_path, 'w') as f: + f.write(lang_search_path_content) + + # patch search file URL in language file + with open(fpath) as f: + content = f.read() + new_content = content.replace( + 'search.html', + f'search_{language}.html', + ) + with open(fpath, 'w') as f: + f.write(new_content) + + +THEME_PATCH_SEARCH_FILES_FUNCS = { + 'readthedocs': _reathedocs_patch_search_files, +} + +## +# Update 'search_index.json#config.lang' only for some themes: +## + +THEME_ALTER_SEARCH_INDEX_LANG_CONFIG = [ + 'material', +] + + +class TranslationsSearchPatcher: + supported_themes = THEME_WORKER_FILES_FUNCS.keys() + + def __init__( + self, + site_dir, + languages, + default_language, + theme_name, + locations, + ): + self.site_dir = site_dir + self.search_index_json_path = os.path.join( + site_dir, + 'search', + 'search_index.json', + ) + + with open(self.search_index_json_path) as f: + self.search_index_json = json.load(f) + + self.search_index = self.search_index_json['docs'] + + self.languages = languages + self.default_language = default_language + self.theme_name = theme_name + + # map from locations of files to languages + self.locations = locations + + # {lang: [records]} + self.lang_search_indexes = {language: [] for language in languages} + + # worker files which load search indexes + self.worker_js_files = THEME_WORKER_FILES_FUNCS[self.theme_name]( + self.site_dir, + ) + for js_file in self.worker_js_files: + if None in js_file.values(): + raise OSError( + 'The search worker file can not be retrieved from' + f' site directory {self.site_dir}', + ) + + def patch_site_dir(self): + # build indexes for languages + for record in self.search_index: + if not record['location']: + self.lang_search_indexes[self.default_language].append(record) + elif '#' in record['location']: + if record['location'].startswith('#'): + self.lang_search_indexes[self.default_language].append( + record, + ) + else: + clean_location = record['location'].split('#')[0] + if clean_location in self.locations: + language = self.locations[clean_location] + self.lang_search_indexes[language].append(record) + elif record['location'] in self.locations: + language = self.locations[record['location']] + self.lang_search_indexes[language].append(record) + + # get site directory HTML files ordered by language + html_files_by_language = self._get_html_files_by_language() + + for language in self.languages: + # create indexes for languages + self._create_lang_search_index_json( + language, + self.lang_search_indexes[language], + ) + + # create javascript assets by language to load custom + # search_index_{language}.json files depending on the + # active theme + + # patch Javascript worker files for the language + THEME_WORKER_PATCHS_FUNCS[self.theme_name]( + self.worker_js_files, + language, + ) + + # patch HTML files to load localized assets + for fpath in html_files_by_language[language]: + THEME_HTML_PATCHS_FUNCS[self.theme_name]( + fpath, + language, + self.worker_js_files, + ) + + # if the theme needs to patch search files, patch them + if self.theme_name in THEME_GET_SEARCH_FILES_FUNCS: + search_files = THEME_GET_SEARCH_FILES_FUNCS[self.theme_name]( + self.site_dir, + ) + for language in self.languages: + for fpath in html_files_by_language[language]: + THEME_PATCH_SEARCH_FILES_FUNCS[self.theme_name]( + fpath, + language, + self.default_language, + search_files, + ) + + def _create_lang_search_index_json(self, language, records): + search_index = copy.copy(self.search_index_json) + search_index['docs'] = records + if ( + self.theme_name in THEME_ALTER_SEARCH_INDEX_LANG_CONFIG + and 'config' in search_index + and 'lang' in search_index['config'] + ): + search_index['config']['lang'] = [language] + + new_path = _language_extension_path( + self.search_index_json_path, + '.json', + language, + ) + with open(new_path, 'w') as f: + f.write(json.dumps(search_index)) + + def _get_html_files_by_language(self): + language_files = {language: [] for language in self.languages} + + for root, _, files in os.walk(self.site_dir): + for fname in [f for f in files if f.endswith('.html')]: + fpath = os.path.join(root, fname) + relpath = os.path.relpath(fpath, self.site_dir) + location = removesuffix(relpath, 'index.html') + if location in self.locations: + language = self.locations[location] + language_files[language].append(fpath) + else: + language_files[self.default_language].append(fpath) + return language_files diff --git a/mkdocs_mdpo_plugin/translations.py b/mkdocs_mdpo_plugin/translations.py index 7475b8f..1c3bb2a 100644 --- a/mkdocs_mdpo_plugin/translations.py +++ b/mkdocs_mdpo_plugin/translations.py @@ -54,6 +54,7 @@ class Translations: 'compendium_msgstrs_tr', 'current', 'all', + 'locations', } def __init__(self): @@ -84,6 +85,11 @@ def __init__(self): # {lang: [Translation(...), Translation(...), ...]} self.all = {} + # possible URL locations of records with its correspondient + # language, needed to discriminate records on search indexes + # {location: language} + self.locations = {} + def __str__(self): # pragma: no cover current = 'None' if self.current is None else 'Translation(...)' return ( @@ -93,6 +99,7 @@ def __str__(self): # pragma: no cover f' compendium_msgids={str(self.compendium_msgids)},' ' compendium_msgstrs_tr=' f'{str(self.compendium_msgstrs_tr)},' - f' current={current}' + f' current={current},' + f' locations={str(self.locations)}' ')' ) diff --git a/setup.cfg b/setup.cfg index b7c3267..1c89591 100644 --- a/setup.cfg +++ b/setup.cfg @@ -1,6 +1,6 @@ [metadata] name = mkdocs_mdpo_plugin -version = 0.0.20 +version = 0.0.21 description = Mkdocs plugin for translations using PO files. long_description = file: README.md long_description_content_type = text/markdown diff --git a/tests/test_examples.py b/tests/test_examples.py new file mode 100644 index 0000000..2f67994 --- /dev/null +++ b/tests/test_examples.py @@ -0,0 +1,34 @@ +import os +import shutil +import subprocess +import sys + +import pytest + + +EXAMPLES_DIR = os.path.join( + os.path.dirname(os.path.dirname(__file__)), + 'examples', +) + + +@pytest.mark.parametrize('example_dirname', os.listdir(EXAMPLES_DIR)) +def test_examples(example_dirname): + example_dirpath = os.path.join(EXAMPLES_DIR, example_dirname) + site_dir = os.path.join(example_dirpath, 'site') + if os.path.isdir(site_dir): + shutil.rmtree(site_dir) + + proc = subprocess.run( + [sys.executable, '-m', 'mkdocs', 'build'], + cwd=example_dirpath, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + ) + + norm_stderr = proc.stderr.decode('utf-8').lower() + assert proc.returncode == 0 + assert 'warning' not in norm_stderr + assert 'error' not in norm_stderr + + assert os.path.isdir(site_dir) diff --git a/tests/test_plugin_config.py b/tests/test_plugin_config.py index c818f62..ec1ee8e 100644 --- a/tests/test_plugin_config.py +++ b/tests/test_plugin_config.py @@ -420,6 +420,18 @@ def test_plugin_config( ), id='languages=undefined-theme=material', ), + pytest.param( + { + 'cross_language_search': False, + }, + {}, + mkdocs.config.base.ValidationError, + ( + '"cross_language_search" setting is disabled but' + ' no "search" plugin has been added to "plugins"' + ), + id='cross_language_search=False-plugins={}', + ), ), ) def test_plugin_config_errors( diff --git a/tests/test_search_indexes.py b/tests/test_search_indexes.py new file mode 100644 index 0000000..09771b9 --- /dev/null +++ b/tests/test_search_indexes.py @@ -0,0 +1,200 @@ +"""Translations search indexes tests.""" + +import json +import os + +import pytest + + +TESTS = ( + pytest.param( # mkdocs theme + { + 'index.md': ( + 'Foo' + ), + }, + { + 'es/index.md.po': { + 'Foo': 'Foo es', + }, + }, + { + 'languages': ['en', 'es'], + 'cross_language_search': False, + }, + { + 'plugins': [ + { + 'search': {}, + }, + ], + }, + { + 'index.html': [ + '

Foo

', + ], + 'es/index.html': [ + '

Foo es

', + ], + 'search/main_en.js': [ + 'worker_en.js', + ], + 'search/main_es.js': [ + 'worker_es.js', + ], + 'search/worker_en.js': [ + 'search_index_en.json', + ], + 'search/worker_es.js': [ + 'search_index_es.json', + ], + }, + id='mkdocs', + ), + pytest.param( # readthedocs theme + { + 'index.md': ( + 'Foo' + ), + }, + { + 'es/index.md.po': { + 'Foo': 'Foo es', + }, + }, + { + 'languages': ['en', 'es'], + 'cross_language_search': False, + }, + { + 'plugins': [ + { + 'search': {}, + }, + ], + 'theme': { + 'name': 'readthedocs', + }, + }, + { + 'index.html': [ + '

Foo

', + 'search_en.html', + ], + 'es/index.html': [ + '

Foo es

', + 'search_es.html', + ], + 'search/main_en.js': [ + 'worker_en.js', + ], + 'search/main_es.js': [ + 'worker_es.js', + ], + 'search/worker_en.js': [ + 'search_index_en.json', + ], + 'search/worker_es.js': [ + 'search_index_es.json', + ], + 'search_en.html': [ + 'search_en.html', + ], + 'search_es.html': [ + 'search_es.html', + ], + }, + id='readthedocs', + ), + pytest.param( # material theme + { + 'index.md': ( + 'Foo' + ), + }, + { + 'es/index.md.po': { + 'Foo': 'Foo es', + }, + }, + { + 'languages': ['en', 'es'], + 'cross_language_search': False, + }, + { + 'plugins': [ + { + 'search': {}, + }, + ], + 'theme': { + 'name': 'material', + }, + }, + { + 'index.html': [ + '

Foo

', + '_en.js', + ], + 'es/index.html': [ + '

Foo es

', + '_es.js', + ], + }, + id='material', + ), +) + + +@pytest.mark.parametrize( + ( + 'input_files_contents', + 'translations', + 'plugin_config', + 'additional_config', + 'expected_output_files', + ), + TESTS, +) +def test_search_indexes( + input_files_contents, + translations, + plugin_config, + additional_config, + expected_output_files, + mkdocs_build, +): + def check_search_indexes(context): + search_dir = os.path.join(context['site_dir'], 'search') + + for fname in os.listdir(search_dir): + if fname.endswith('index.json') or not fname.endswith('.json'): + continue + fpath = os.path.join(search_dir, fname) + with open(fpath) as f: + search_index = json.loads(f.read()) + + language = fname.split('_')[-1].split('.')[0] + + assert isinstance(search_index, dict) + assert isinstance(language, str) + assert len(search_index) > 0 + + if language == plugin_config['languages'][0]: # en + other_language = plugin_config['languages'][1] # es + for record in search_index['docs']: + assert not record['location'].startswith( + (f'{language}/', f'{other_language}/'), + ) + else: + for record in search_index['docs']: + assert record['location'].startswith(f'{language}/') + + mkdocs_build( + input_files_contents, + translations, + plugin_config, + additional_config, + expected_output_files, + callback_after_first_build=check_search_indexes, + )