ninja backend: don't rebuild all tests by default

When running `ninja all` we shouldn't build testsuite programs as these
might not be wanted e.g. in order to just install the project. We do
want them to be built when running `ninja test`. Since meson 0.63 we
actually have a dedicated ninja alias for test dependencies -- move
these from the "all" rule to the dedicated test/benchmark rules.

docs/markdown/snippets/test_dependencies.md

@@ -0,0 +1,19 @@
+## Test targets no longer built by default
+
+The `ninja test` / `meson test` rules have been reworked to no longer force
+tests to be built unnecessarily. The correct, guaranteed workflow for this has
+been to either run `ninja test` or `meson test`, both of which rebuild
+dependencies on demand. *Also* building test-only binaries as part of
+installing the project (`ninja && ninja install`) was unnecessary and has no
+use case.
+
+Some users might have been relying on the "all" target building test
+dependencies in combination with `meson test --no-rebuild` in order to skip
+calling out to ninja when running tests. The `--no-rebuild` option has always
+been intended for expert usage -- you must provide your own guarantees that it
+will work -- and it should be noted that this change means test programs are no
+longer guaranteed to have been built, depending on whether those test programs
+were *also* defined to build by default / marked as installable. The desired
+behavior of building test programs in a separate stage can be restored by
+building `ninja all meson-test-prereq` (or `meson-benchmark-prereq` for running
+benchmarks), as these prereq targets have been available since meson 0.63.0.

mesonbuild/backend/ninjabackend.py

@@ -1326,7 +1326,7 @@ def generate_tests(self) -> None:
             cmd += ['--no-stdsplit']
         if self.environment.coredata.get_option(OptionKey('errorlogs')):
             cmd += ['--print-errorlogs']
-        elem = self.create_phony_target('test', 'CUSTOM_COMMAND', ['all', 'PHONY'])
+        elem = self.create_phony_target('test', 'CUSTOM_COMMAND', ['all', 'meson-test-prereq', 'PHONY'])
         elem.add_item('COMMAND', cmd)
         elem.add_item('DESC', 'Running all tests')
         elem.add_item('pool', 'console')
@@ -1336,7 +1336,7 @@ def generate_tests(self) -> None:
         cmd = self.environment.get_build_command(True) + [
             'test', '--benchmark', '--logbase',
             'benchmarklog', '--num-processes=1', '--no-rebuild']
-        elem = self.create_phony_target('benchmark', 'CUSTOM_COMMAND', ['all', 'PHONY'])
+        elem = self.create_phony_target('benchmark', 'CUSTOM_COMMAND', ['all', 'meson-benchmark-prereq', 'PHONY'])
         elem.add_item('COMMAND', cmd)
         elem.add_item('DESC', 'Running benchmark suite')
         elem.add_item('pool', 'console')
@@ -3718,10 +3718,6 @@ def generate_ending(self) -> None:
                 ('meson-test-prereq', self.get_testlike_targets()),
                 ('meson-benchmark-prereq', self.get_testlike_targets(True))]:
             targetlist = []
-            # These must also be built by default.
-            # XXX: Sometime in the future these should be built only before running tests.
-            if targ == 'all':
-                targetlist.extend(['meson-test-prereq', 'meson-benchmark-prereq'])
             for t in deps.values():
                 # Add the first output of each target to the 'all' target so that
                 # they are all built

run_project_tests.py

@@ -712,7 +712,14 @@ def _run_test(test: TestDef,
     # Build with subprocess
     def build_step() -> None:
         build_start = time.time()
-        pc, o, _ = Popen_safe(compile_commands + dir_args, cwd=test_build_dir, stderr=subprocess.STDOUT)
+
+        if backend is Backend.ninja:
+            # FIXME: meson test inprocess does not handle running ninja via StringIO
+            targets = ['all', 'meson-test-prereq', 'meson-benchmark-prereq']
+        else:
+            targets = []
+
+        pc, o, _ = Popen_safe(compile_commands + dir_args + targets, cwd=test_build_dir, stderr=subprocess.STDOUT)
         testresult.add_step(BuildStep.build, o, '', '', time.time() - build_start)
         if should_fail == 'build':
             if pc.returncode != 0: