Allow to ignore duration in tests (#3569)

There are other ways to adjust this value, but sometimes
it is just easier to just let test run as long as they need.

Co-authored-by: Petr Šplíchal <[email protected]>

Author: lukaszachy

docs/overview.rst

@@ -629,6 +629,8 @@ TMT_TEST_PIDFILE_ROOT
     of temporary directory permissions, e.g. ``chmod 1777``, to
     allow access to users with all privilege levels.
 
+.. _plugin-variables:
+
 Plugin Variables
 ----------------
 
@@ -685,6 +687,9 @@ example, an interactive mode would be enabled in this run::
       # Here the environment variable will take effect:
       $ TMT_PLUGIN_DISCOVER_FMF_VERBOSE=2 tmt run -a discover -h fmf ...
 
+   Several plugins (``report -h reportportal``, ``report -h polarion``,
+   ``execute -h tmt``) allow selected variables to be processed,
+   even when plugin is not specified on the command line.
 
 .. _regular-expressions:
 

docs/releases.rst

@@ -8,13 +8,19 @@
 tmt-1.44.0
 ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
 
+
 The ``results.yaml`` file is now populated with test results
 right after the ``discover`` step is finished and the file is
 continuously updated during test execution to provide the latest
 results. This change also adds a new ``pending`` result outcome
 to the :ref:`/spec/results` specification for tests that were
 discovered but not yet executed.
 
+Execute tmt option ``--ignore-duration`` makes tmt to execute
+the test as long as it needs. Execute plugin doesn't need to be
+specified on the commandline for :ref:`plugin-variables` to work
+for this option.
+
 
 tmt-1.43.0
 ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~

tests/execute/ignore-duration/data/.fmf/version

@@ -0,0 +1 @@
+1

tests/execute/ignore-duration/data/demo.fmf

@@ -0,0 +1,18 @@
+/test:
+    test: sleep 4
+    duration: "1"
+
+/plan:
+    discover:
+        how: fmf
+    provision:
+        how: local
+    execute:
+        how: tmt
+    /no-option:
+    /via-plan-true:
+        execute+:
+            ignore-duration: true
+    /via-plan-false:
+        execute+:
+            ignore-duration: false

tests/execute/ignore-duration/main.fmf

@@ -0,0 +1,2 @@
+summary: Verify that the ignore-duration is correctly handled
+duration: 5m

tests/execute/ignore-duration/test.sh

@@ -0,0 +1,45 @@
+#!/bin/bash
+. /usr/share/beakerlib/beakerlib.sh || exit 1
+
+
+rlJournalStart
+    rlPhaseStartSetup
+        rlRun "tmp=\$(mktemp -d)" 0 "Creating tmp directory"
+        rlRun "export TMT_WORKDIR_ROOT=$tmp"
+        rlRun "pushd data"
+    rlPhaseEnd
+
+    rlPhaseStartTest "No envvar used"
+        rlRun -s "tmt run -vv plan -n /no-option" "2"
+        rlAssertGrep 'errr /demo/test (timeout)' $rlRun_LOG '-F'
+
+        rlRun "tmt run -vv plan -n /via-plan-true" "0"
+
+        rlRun "tmt run -vv plan -n /via-plan-false" "2"
+        rlAssertGrep 'errr /demo/test (timeout)' $rlRun_LOG '-F'
+    rlPhaseEnd
+
+    rlPhaseStartTest "With IGNORE_DURATION=1"
+        export TMT_PLUGIN_EXECUTE_TMT_IGNORE_DURATION=1
+        rlRun "tmt run -vv plan -n /no-option"
+        rlRun "tmt run -vv plan -n /via-plan-true"
+        # ENV should win over CLI or file values, but to be consistent with
+        # reporportal/polarion plugin envar is weaker than plan.fmf
+        rlRun "tmt run -vv plan -n /via-plan-false" "2"
+    rlPhaseEnd
+
+    rlPhaseStartTest "With IGNORE_DURATION=0"
+        export TMT_PLUGIN_EXECUTE_TMT_IGNORE_DURATION=0
+        rlRun "tmt run -vv plan -n /no-option" "2"
+        # ENV should win over CLI or file values, but to be consistent with
+        # reporportal/polarion plugin envar is weaker than plan.fmf
+        rlRun "tmt run -vv plan -n /via-plan-true"
+        rlRun "tmt run -vv plan -n /via-plan-false" "2"
+    rlPhaseEnd
+
+
+    rlPhaseStartCleanup
+        rlRun "popd"
+        rlRun "rm -rf $tmp" 0 "Removing tmp directory"
+    rlPhaseEnd
+rlJournalEnd

tmt/schemas/execute/tmt.yaml

@@ -23,6 +23,9 @@ properties:
   exit-first:
     type: boolean
 
+  ignore-duration:
+    type: boolean
+
   # name attribute can exist if more methods
   name:
     type: string

tmt/steps/execute/__init__.py

@@ -31,6 +31,7 @@
     Path,
     ShellScript,
     Stopwatch,
+    configure_bool_constant,
     format_duration,
     format_timestamp,
 )
@@ -261,6 +262,18 @@ class ExecuteStepData(tmt.steps.WhereableStepData, tmt.steps.StepData):
         option='--duration',
         help='The maximal time allowed for the test to run.',
     )
+    ignore_duration: bool = field(
+        default=configure_bool_constant(False, 'TMT_PLUGIN_EXECUTE_TMT_IGNORE_DURATION'),
+        option='--ignore-duration',
+        is_flag=True,
+        envvar="TMT_PLUGIN_EXECUTE_TMT_IGNORE_DURATION",
+        help="""
+             Ignore test duration value and allow test to run forever.
+             Can be set by environment variable even when step is not
+             specified on the commandline. This environment variable
+             will be replaced by fmf config file or CLI arguments.
+             """,
+    )
     exit_first: bool = field(
         default=False,
         option=('-x', '--exit-first'),
@@ -709,6 +722,7 @@ def go(
         logger: tmt.log.Logger,
     ) -> None:
         self.go_prolog(logger)
+        logger.verbose('ignore-duration', self.data.ignore_duration, 'green', level=2)
         logger.verbose('exit-first', self.data.exit_first, 'green', level=2)
 
     @property

tmt/steps/execute/internal.py

@@ -560,6 +560,10 @@ def _save_process(
 
                 timeout = None
 
+            elif self.data.ignore_duration:
+                logger.debug("Test duration is not effective due ignore-duration option.")
+                timeout = None
+
             else:
                 timeout = tmt.utils.duration_to_seconds(
                     test.duration, tmt.base.DEFAULT_TEST_DURATION_L1

tmt/utils/__init__.py

@@ -120,6 +120,24 @@ def configure_constant(default: int, envvar: str) -> int:
         ) from exc
 
 
+def configure_bool_constant(default: bool, envvar: str) -> bool:
+    """
+    Deduce the bool value of global constant.
+
+    Value '1' means True, all other values mean False.
+
+    :param default: the default value of the constant.
+    :param envvar: name of the optional environment variable which would
+        override the default value.
+    :returns: value extracted from the environment variable, or the
+        given default value if the variable did not exist.
+    """
+    value = os.environ.get(envvar)
+    if value is None:
+        return default
+    return value == "1"
+
+
 log = fmf.utils.Logging('tmt').logger