New xgettext method to i18n module

docs/markdown/i18n-module.md

@@ -74,3 +74,50 @@ for normal keywords. In addition it accepts these keywords:
 * `mo_targets` *required*: mo file generation targets as returned by `i18n.gettext()`.
 
 *Added 0.62.0*
+
+
+### i18n.xgettext()
+
+``` meson
+i18n.xgettext(name, sources..., args: [...], recursive: false)
+```
+
+Invokes the `xgettext` program on given sources, to generate a `.pot` file.
+This function is to be used when the `gettext` function workflow it not suitable
+for your project. For example, it can be used to produce separate `.pot` files
+for each executable.
+
+Positional arguments are the following:
+
+* name `str`: the name of the resulting pot file.
+* sources `list[str|File|build_tgt|custom_tgt]`:
+          source files or targets. May be a list of `string`, `File`, [[@build_tgt]],
+          or [[@custom_tgt]] returned from other calls to this function.
+
+Keyword arguments are the following:
+
+- recursive `bool`:
+        if `true`, will merge the resulting pot file with extracted pot files
+        related to dependencies of the given source targets. For instance,
+        if you build an executable, then you may want to merge the executable
+        translations with the translations from the dependent libraries.
+- install `bool`: if `true`, will add the resulting pot file to install targets.
+- install_tag `str`: install tag to use for the install target.
+- install_dir `str`: directory where to install the resulting pot file.
+
+The `i18n.xgettext()` function returns a [[@custom_tgt]].
+
+Usually, you want to pass one build target as sources, and the list of header files
+for that target. If the number of source files would result in a command line that
+is too long, the list of source files is written to a file at config time, to be
+used as input for the `xgettext` program.
+
+The `recursive: true` argument is to be given to targets that will actually read
+the resulting `.mo` file. Each time you call the `i18n.xgettext()` function,
+it maps the source targets to the resulting pot file. When `recursive: true` is
+given, all generated pot files from dependencies of the source targets are
+included to generate the final pot file. Therefore, adding a dependency to
+source target will automatically add the translations of that dependency to the
+needed translations for that source target.
+
+*Added 1.7.0*

docs/markdown/snippets/i18n_xgettext.md

@@ -0,0 +1,12 @@
+## i18n module xgettext
+
+There is a new `xgettext` function in `i18n` module that acts as a
+wrapper around `xgettext`. It allows to extract strings to translate from
+source files.
+
+This function is convenient, because:
+- It can find the sources files from a build target;
+- It will use an intermediate file when the number of source files is too
+  big to be handled directly from the command line;
+- It is able to get strings to translate from the dependencies of the given
+  targets.

mesonbuild/backend/ninjabackend.py

@@ -91,27 +91,8 @@ def gcc_rsp_quote(s: str) -> str:
 
     return quote_func(s)
 
-def get_rsp_threshold() -> int:
-    '''Return a conservative estimate of the commandline size in bytes
-    above which a response file should be used.  May be overridden for
-    debugging by setting environment variable MESON_RSP_THRESHOLD.'''
-
-    if mesonlib.is_windows():
-        # Usually 32k, but some projects might use cmd.exe,
-        # and that has a limit of 8k.
-        limit = 8192
-    else:
-        # Unix-like OSes usually have very large command line limits, (On Linux,
-        # for example, this is limited by the kernel's MAX_ARG_STRLEN). However,
-        # some programs place much lower limits, notably Wine which enforces a
-        # 32k limit like Windows. Therefore, we limit the command line to 32k.
-        limit = 32768
-    # Be conservative
-    limit = limit // 2
-    return int(os.environ.get('MESON_RSP_THRESHOLD', limit))
-
 # a conservative estimate of the command-line length limit
-rsp_threshold = get_rsp_threshold()
+rsp_threshold = mesonlib.get_rsp_threshold()
 
 # ninja variables whose value should remain unquoted. The value of these ninja
 # variables (or variables we use them in) is interpreted directly by ninja

mesonbuild/build.py

@@ -1324,7 +1324,7 @@ def get_dependencies_recurse(self, result: OrderedSet[BuildTargetTypes], include
     def get_source_subdir(self):
         return self.subdir
 
-    def get_sources(self):
+    def get_sources(self) -> T.List[File]:
         return self.sources
 
     def get_objects(self) -> T.List[T.Union[str, 'File', 'ExtractedObjects']]:
@@ -2551,7 +2551,7 @@ def __init__(self, shared: SharedLibrary, static: StaticLibrary, preferred_libra
     def __repr__(self) -> str:
         return f'<BothLibraries: static={repr(self.static)}; shared={repr(self.shared)}>'
 
-    def get(self, lib_type: T.Literal['static', 'shared']) -> LibTypes:
+    def get(self, lib_type: T.Literal['static', 'shared']) -> T.Union[StaticLibrary, SharedLibrary]:
         if lib_type == 'static':
             return self.static
         if lib_type == 'shared':

mesonbuild/modules/i18n.py

@@ -4,6 +4,7 @@
 from __future__ import annotations
 
 from os import path
+from pathlib import Path
 import shlex
 import typing as T
 
@@ -13,7 +14,8 @@
 from ..options import OptionKey
 from .. import mlog
 from ..interpreter.type_checking import CT_BUILD_BY_DEFAULT, CT_INPUT_KW, INSTALL_TAG_KW, OUTPUT_KW, INSTALL_DIR_KW, INSTALL_KW, NoneType, in_set_validator
-from ..interpreterbase import FeatureNew, InvalidArguments
+from ..interpreterbase import FeatureNew
+from ..interpreterbase.exceptions import InvalidArguments
 from ..interpreterbase.decorators import ContainerTypeInfo, KwargInfo, noPosargs, typed_kwargs, typed_pos_args
 from ..programs import ExternalProgram
 from ..scripts.gettext import read_linguas
@@ -65,6 +67,16 @@ class ItsJoinFile(TypedDict):
         its_files: T.List[str]
         mo_targets: T.List[T.Union[build.BuildTarget, build.CustomTarget, build.CustomTargetIndex]]
 
+    class XgettextProgramT(TypedDict):
+
+        args: T.List[str]
+        recursive: bool
+        install: bool
+        install_dir: T.Optional[str]
+        install_tag: T.Optional[str]
+
+    SourcesType = T.Union[str, mesonlib.File, build.BuildTarget, build.BothLibraries, build.CustomTarget]
+
 
 _ARGS: KwargInfo[T.List[str]] = KwargInfo(
     'args',
@@ -115,6 +127,124 @@ class ItsJoinFile(TypedDict):
 }
 
 
+class XgettextProgram:
+
+    pot_files: T.Dict[str, build.CustomTarget] = {}
+
+    def __init__(self, xgettext: ExternalProgram, interpreter: Interpreter):
+        self.xgettext = xgettext
+        self.interpreter = interpreter
+
+    def extract(self,
+                name: str,
+                sources: T.List[SourcesType],
+                args: T.List[str],
+                recursive: bool,
+                install: bool,
+                install_dir: T.Optional[str],
+                install_tag: T.Optional[str]) -> build.CustomTarget:
+
+        if not name.endswith('.pot'):
+            name += '.pot'
+
+        source_files = self._get_source_files(sources)
+
+        command = self.xgettext.command + args
+        command.append(f'--directory={self.interpreter.environment.get_source_dir()}')
+        command.append(f'--directory={self.interpreter.environment.get_build_dir()}')
+        command.append('--output=@OUTPUT@')
+
+        depends = list(self._get_depends(sources)) if recursive else []
+        rsp_file = self._get_rsp_file(name, source_files, depends, command)
+        inputs: T.List[T.Union[mesonlib.File, build.CustomTarget]]
+        if rsp_file:
+            inputs = [rsp_file]
+            depend_files = list(source_files)
+            command.append('--files-from=@INPUT@')
+        else:
+            inputs = list(source_files) + depends
+            depends = None
+            depend_files = None
+            command.append('@INPUT@')
+
+        ct = build.CustomTarget(
+            '',
+            self.interpreter.subdir,
+            self.interpreter.subproject,
+            self.interpreter.environment,
+            command,
+            inputs,
+            [name],
+            depend_files = depend_files,
+            extra_depends = depends,
+            install = install,
+            install_dir = [install_dir] if install_dir else None,
+            install_tag = [install_tag] if install_tag else None,
+            description = 'Extracting translations to {}',
+        )
+
+        for source_id in self._get_source_id(sources):
+            self.pot_files[source_id] = ct
+        self.pot_files[ct.get_id()] = ct
+
+        self.interpreter.add_target(ct.name, ct)
+        return ct
+
+    def _get_source_files(self, sources: T.Iterable[SourcesType]) -> T.Set[mesonlib.File]:
+        source_files = set()
+        for source in sources:
+            if isinstance(source, mesonlib.File):
+                source_files.add(source)
+            elif isinstance(source, str):
+                mesonlib.check_direntry_issues(source)
+                source_files.add(mesonlib.File.from_source_file(self.interpreter.source_root, self.interpreter.subdir, source))
+            elif isinstance(source, build.BuildTarget):
+                source_files.update(source.get_sources())
+            elif isinstance(source, build.BothLibraries):
+                source_files.update(source.get('shared').get_sources())
+        return source_files
+
+    def _get_depends(self, sources: T.Iterable[SourcesType]) -> T.Set[build.CustomTarget]:
+        depends = set()
+        for source in sources:
+            if isinstance(source, build.BuildTarget):
+                for source_id in self._get_source_id(source.get_dependencies()):
+                    if source_id in self.pot_files:
+                        depends.add(self.pot_files[source_id])
+            elif isinstance(source, build.CustomTarget):
+                # Dependency on another extracted pot file
+                source_id = source.get_id()
+                if source_id in self.pot_files:
+                    depends.add(self.pot_files[source_id])
+        return depends
+
+    def _get_rsp_file(self,
+                      name: str,
+                      source_files: T.Iterable[mesonlib.File],
+                      depends: T.Iterable[build.CustomTarget],
+                      arguments: T.List[str]) -> T.Optional[mesonlib.File]:
+        source_list = '\n'.join(source.relative_name() for source in source_files)
+        for dep in depends:
+            source_list += '\n' + path.join(dep.subdir, dep.get_filename())
+
+        estimated_cmdline_length = len(source_list) + sum(len(arg) + 1 for arg in arguments) + 1
+        if estimated_cmdline_length < mesonlib.get_rsp_threshold():
+            return None
+
+        rsp_file = Path(self.interpreter.environment.build_dir, self.interpreter.subdir, name+'.rsp')
+        rsp_file.write_text(source_list, encoding='utf-8')
+
+        return mesonlib.File.from_built_file(self.interpreter.subdir, rsp_file.name)
+
+    def _get_source_id(self, sources: T.Iterable[T.Union[SourcesType, build.CustomTargetIndex]]) -> T.Iterable[str]:
+        for source in sources:
+            if isinstance(source, build.Target):
+                yield source.get_id()
+            elif isinstance(source, build.BothLibraries):
+                yield source.get('static').get_id()
+                yield source.get('shared').get_id()
+
+
 class I18nModule(ExtensionModule):
 
     INFO = ModuleInfo('i18n')
@@ -125,6 +255,7 @@ def __init__(self, interpreter: 'Interpreter'):
             'merge_file': self.merge_file,
             'gettext': self.gettext,
             'itstool_join': self.itstool_join,
+            'xgettext': self.xgettext,
         })
         self.tools: T.Dict[str, T.Optional[T.Union[ExternalProgram, build.Executable]]] = {
             'itstool': None,
@@ -398,6 +529,27 @@ def itstool_join(self, state: 'ModuleState', args: T.List['TYPE_var'], kwargs: '
 
         return ModuleReturnValue(ct, [ct])
 
+    @FeatureNew('i18n.xgettext', '1.7.0')
+    @typed_pos_args('i18n.xgettext', str, varargs=(str, mesonlib.File, build.BuildTarget, build.BothLibraries, build.CustomTarget), min_varargs=1)
+    @typed_kwargs(
+        'i18n.xgettext',
+        _ARGS,
+        KwargInfo('recursive', bool, default=False),
+        INSTALL_KW,
+        INSTALL_DIR_KW,
+        INSTALL_TAG_KW,
+    )
+    def xgettext(self, state: ModuleState, args: T.Tuple[str, T.List[SourcesType]], kwargs: XgettextProgramT) -> build.CustomTarget:
+        toolname = 'xgettext'
+        if self.tools[toolname] is None or not self.tools[toolname].found():
+            self.tools[toolname] = state.find_program(toolname, required=True, for_machine=mesonlib.MachineChoice.BUILD)
+
+        if kwargs['install'] and not kwargs['install_dir']:
+            raise InvalidArguments('XgettextProgram.extract: "install_dir" keyword argument must be set when "install" is true.')
+
+        xgettext_program = XgettextProgram(T.cast('ExternalProgram', self.tools[toolname]), self.interpreter)
+        return xgettext_program.extract(*args, **kwargs)
+
 
 def initialize(interp: 'Interpreter') -> I18nModule:
     return I18nModule(interp)

mesonbuild/utils/universal.py

@@ -106,6 +106,7 @@ class _VerPickleLoadable(Protocol):
     'generate_list',
     'get_compiler_for_source',
     'get_filenames_templates_dict',
+    'get_rsp_threshold',
     'get_variable_regex',
     'get_wine_shortpath',
     'git',
@@ -2369,6 +2370,27 @@ def first(iter: T.Iterable[_T], predicate: T.Callable[[_T], bool]) -> T.Optional
     return None
 
 
+def get_rsp_threshold() -> int:
+    '''Return a conservative estimate of the commandline size in bytes
+    above which a response file should be used.  May be overridden for
+    debugging by setting environment variable MESON_RSP_THRESHOLD.'''
+
+    if is_windows():
+        # Usually 32k, but some projects might use cmd.exe,
+        # and that has a limit of 8k.
+        limit = 8192
+    else:
+        # Unix-like OSes usually have very large command line limits, (On Linux,
+        # for example, this is limited by the kernel's MAX_ARG_STRLEN). However,
+        # some programs place much lower limits, notably Wine which enforces a
+        # 32k limit like Windows. Therefore, we limit the command line to 32k.
+        limit = 32768
+
+    # Be conservative
+    limit = limit // 2
+    return int(os.environ.get('MESON_RSP_THRESHOLD', limit))
+
+
 class lazy_property(T.Generic[_T]):
     """Descriptor that replaces the function it wraps with the value generated.
 

test cases/frameworks/38 gettext extractor/meson.build

@@ -0,0 +1,15 @@
+project(
+    'gettext extractor',
+    'c',
+    default_options: {'default_library': 'static'},
+    meson_version: '1.8.0',
+)
+
+if not find_program('xgettext', required: false).found()
+    error('MESON_SKIP_TEST xgettext command not found')
+endif
+
+i18n = import('i18n')
+xgettext_args = ['-ktr', '--add-comments=TRANSLATOR:', '--from-code=UTF-8']
+
+subdir('src')

test cases/frameworks/38 gettext extractor/src/lib1/lib1.c

@@ -0,0 +1,10 @@
+#include "lib1.h"
+
+#include <stdio.h>
+
+#define tr(STRING) (STRING)
+
+void say_something(void)
+{
+    printf("%s\n", tr("Something!"));
+}

test cases/frameworks/38 gettext extractor/src/lib1/lib1.h

@@ -0,0 +1,6 @@
+#ifndef LIB1_H
+#define LIB1_H
+
+void say_something(void);
+
+#endif

test cases/frameworks/38 gettext extractor/src/lib1/meson.build

@@ -0,0 +1,3 @@
+lib1 = library('mylib1', 'lib1.c')
+lib1_pot = i18n.xgettext('lib1', lib1, args: xgettext_args)
+lib1_includes = include_directories('.')

test cases/frameworks/38 gettext extractor/src/lib2/lib2.c

@@ -0,0 +1,13 @@
+#include "lib2.h"
+
+#include <lib1.h>
+
+#include <stdio.h>
+
+#define tr(STRING) (STRING)
+
+void say_something_else(void)
+{
+    say_something();
+    printf("%s\n", tr("Something else!"));
+}