#385 Add limits

bin/config.py

@@ -126,11 +126,6 @@ def set_default_args():
 DEFAULT_TIMEOUT = 30
 DEFAULT_INTERACTION_TIMEOUT = 60
 
-
-def get_timeout():
-    return args.timeout or DEFAULT_TIMEOUT
-
-
 # Randomly generated uuid4 for BAPCtools
 BAPC_UUID = '8ee7605a-d1ce-47b3-be37-15de5acd757e'
 BAPC_UUID_PREFIX = 8

bin/generate.py

@@ -226,7 +226,9 @@ def run(self, bar, cwd):
         ans_path = cwd / 'testcase.ans'
 
         # No {name}/{seed} substitution is done since all IO should be via stdin/stdout.
-        result = self.program.run(in_path, ans_path, args=self.args, cwd=cwd, default_timeout=True)
+        result = self.program.run(
+            in_path, ans_path, args=self.args, cwd=cwd, generator_timeout=True
+        )
 
         if result.status == ExecStatus.TIMEOUT:
             bar.debug(f'{Style.RESET_ALL}-> {shorten_path(self.problem, cwd)}')
@@ -710,6 +712,15 @@ def validate_ans(t, problem, testcase, meta_yaml, bar):
                 multipass = 'multipass ' if problem.multipass else ''
                 bar.warn(f'.ans file for {interactive}{multipass}problem is expected to be empty.')
         else:
+            size = ansfile.stat().st_size
+            if (
+                size <= problem.limits.output * 1024 * 1024
+                and problem.limits.output * 1024 * 1024 < 2 * size
+            ):  # we already warn if the limit is exceeded
+                bar.warn(
+                    f'.ans file is {size / 1024 / 1024:.3f}MiB, which is close to output limit (set limits.output to at least {(2*size + 1024 * 1024 - 1) // 1024 // 1024}MiB in problem.yaml)'
+                )
+
             answer_validator_hashes = {
                 **testcase.validator_hashes(validate.AnswerValidator, bar),
                 **testcase.validator_hashes(validate.OutputValidator, bar),

bin/interactive.py

@@ -35,18 +35,17 @@ def run_interactive_testcase(
     submission_args: Optional[list[str]] = None,
 ):
     output_validators = run.problem.validators(validate.OutputValidator)
-    if output_validators is False:
-        fatal('No output validator found!')
-
-    assert len(output_validators) == 1
+    if len(output_validators) != 1:
+        return None
     output_validator = output_validators[0]
 
     # Set limits
-    validator_timeout = config.DEFAULT_INTERACTION_TIMEOUT
+    validation_time = run.problem.limits.validation_time
+    validation_memory = run.problem.limits.validation_memory
 
-    memory_limit = get_memory_limit()
     timelimit = run.problem.settings.timelimit
     timeout = run.problem.settings.timeout
+    memory = run.problem.limits.memory
 
     # Validator command
     def get_validator_command():
@@ -108,10 +107,11 @@ def get_validator_command():
                 stderr=subprocess.PIPE if team_error is False else None,
                 cwd=submission_dir,
                 timeout=timeout,
+                memory=memory,
             )
 
             # Wait
-            (validator_out, validator_err) = validator_process.communicate()
+            (validator_out, validator_err) = validator_process.communicate(timeout=validation_time)
 
             tend = time.monotonic()
             max_duration = max(max_duration, tend - tstart)
@@ -223,7 +223,7 @@ def get_validator_command():
             stderr=subprocess.PIPE if validator_error is False else None,
             cwd=validator_dir,
             pipesize=BUFFER_SIZE,
-            preexec_fn=limit_setter(validator_command, validator_timeout, None, 0),
+            preexec_fn=limit_setter(validator_command, validation_time, validation_memory, 0),
         )
         validator_pid = validator.pid
         # add all programs to the same group (for simplicity we take the pid of the validator)
@@ -259,7 +259,7 @@ def get_validator_command():
             stderr=subprocess.PIPE if team_error is False else None,
             cwd=submission_dir,
             pipesize=BUFFER_SIZE,
-            preexec_fn=limit_setter(submission_command, timeout, memory_limit, gid),
+            preexec_fn=limit_setter(submission_command, timeout, memory, gid),
         )
         submission_pid = submission.pid
 
@@ -275,7 +275,7 @@ def kill_handler_function():
                 os.kill(submission_pid, signal.SIGKILL)
             except ProcessLookupError:
                 pass
-            if validator_timeout > timeout and stop_kill_handler.wait(validator_timeout - timeout):
+            if validation_time > timeout and stop_kill_handler.wait(validation_time - timeout):
                 return
             os.killpg(gid, signal.SIGKILL)
 

bin/problem.py

@@ -45,6 +45,7 @@ def __init__(self, path: Path, tmpdir: Path, label: Optional[str] = None):
         self._validators_cache = dict[  # The "bool" is for "check_constraints"
             tuple[Type[validate.AnyValidator], bool], list[validate.AnyValidator]
         ]()
+        self._validators_warn_cache = set[tuple[Type[validate.AnyValidator], bool]]()
         self._programs = dict[Path, "Program"]()
         self._program_callbacks = dict[Path, list[Callable[["Program"], None]]]()
         # Dictionary from path to parsed file contents.
@@ -125,12 +126,23 @@ def _read_settings(self):
             'uuid': None,
             'limits': {},
         }
-        # TODO: implement other limits
         # TODO move timelimit to this?
+        # defaults from https://github.com/Kattis/problem-package-format/blob/master/spec/legacy-icpc.md#limits
         limits = {
             'time_multiplier': 2,
             'time_safety_margin': 1.5,
             'time_resolution': 1.0,
+            'memory': 2048,  # in MiB
+            'output': 8,  # in MiB
+            'code': 128,  # in KiB
+            'compilation_time': 60,  # in seconds
+            'compilation_memory': 2048,  # in MiB
+            'validation_time': 60,  # in seconds
+            'validation_memory': 2048,  # in MiB
+            # 'validation_output': 8,  # in MiB
+            # BAPCtools extension:
+            'generator_time': config.DEFAULT_TIMEOUT,  # in seconds
+            'visualizer_time': config.DEFAULT_TIMEOUT,  # in seconds
         }
 
         yaml_path = self.path / 'problem.yaml'
@@ -209,6 +221,11 @@ def _read_settings(self):
         self.settings.timeout = int(
             config.args.timeout or self.limits.time_safety_margin * self.settings.timelimit + 1
         )
+        self.limits.validation_time = config.args.timeout or self.limits.validation_time
+        self.limits.generator_time = config.args.timeout or self.limits.generator_time
+        self.limits.visualizer_time = config.args.timeout or self.limits.visualizer_time
+        self.limits.memory = config.args.memory or self.limits.memory
+        self.limits.validation_memory = config.args.memory or self.limits.validation_memory
 
     def _parse_testdata_yaml(p, path, bar):
         assert path.is_relative_to(p.path / 'data')
@@ -556,12 +573,28 @@ def validators(
             singleton list(OutputValidator) if cls is OutputValidator
             list(Validator) otherwise, maybe empty
         """
+        validators = problem._validators(cls, check_constraints)
         if not strict and cls == validate.AnswerValidator:
-            return problem._validators(cls, check_constraints) + problem._validators(
-                validate.OutputValidator, check_constraints
-            )
-        else:
-            return problem._validators(cls, check_constraints)
+            validators += problem._validators(validate.OutputValidator, check_constraints)
+
+        # Check that the proper number of validators is present
+        # do this after handling the strict flag but dont warn every time
+        key = (cls, check_constraints)
+        if key not in problem._validators_warn_cache:
+            problem._validators_warn_cache.add(key)
+            match cls, len(validators):
+                case validate.InputValidator, 0:
+                    warn(f'No input validators found.')
+                case validate.AnswerValidator, 0:
+                    warn(f"No answer validators found")
+                case validate.OutputValidator, l if l != 1:
+                    error(f'Found {len(validators)} output validators, expected exactly one.')
+
+        build_ok = all(v.ok for v in validators)
+
+        # All validators must build.
+        # TODO Really? Why not at least return those that built?
+        return validators if build_ok else []
 
     def _validators(
         problem, cls: Type[validate.AnyValidator], check_constraints=False
@@ -570,7 +603,6 @@ def _validators(
         key = (cls, check_constraints)
         if key in problem._validators_cache:
             return problem._validators_cache[key]
-        ok = True
 
         assert hasattr(cls, 'source_dirs')
         paths = [p for source_dir in cls.source_dirs for p in glob(problem.path / source_dir, '*')]
@@ -581,17 +613,6 @@ def _validators(
                 error("Validation is default but custom output validator exists (ignoring it)")
             paths = [config.tools_root / 'support' / 'default_output_validator.cpp']
 
-        # Check that the proper number of validators is present
-        match cls, len(paths):
-            case validate.InputValidator, 0:
-                warn(f'No input validators found.')
-            case validate.AnswerValidator, 0:
-                log(f"No answer validators found")
-            case validate.OutputValidator, l if l != 1:
-                print(cls)
-                error(f'Found {len(paths)} output validators, expected exactly one.')
-                ok = False
-
         # TODO: Instead of checking file contents, maybe specify this in generators.yaml?
         def has_constraints_checking(f):
             if not f.is_file():
@@ -624,23 +645,16 @@ def has_constraints_checking(f):
             for path in paths
         ]
         bar = ProgressBar(f'Building {cls.validator_type} validator', items=validators)
-        build_ok = True
 
         def build_program(p):
-            nonlocal build_ok
             localbar = bar.start(p)
-            build_ok &= p.build(localbar)
+            p.build(localbar)
             localbar.done()
 
         parallel.run_tasks(build_program, validators)
-
         bar.finalize(print_done=False)
 
-        # All validators must build.
-        # TODO Really? Why not at least return those that built?
-        result = validators if ok and build_ok else []
-
-        problem._validators_cache[key] = result
+        problem._validators_cache[key] = validators
         return validators
 
     # get all testcses and submissions and prepare the output validator

bin/program.py

@@ -95,6 +95,12 @@ def sanitizer():
 # - language:       the detected language
 # - env:            the environment variables used for compile/run command substitution
 # - hash:           a hash of all of the program including all source files
+# - limits          a dict of the optional limts, keys are:
+#                   - code
+#                   - compilation_time
+#                   - compilation_memory
+#                   - timeout
+#                   - memory
 #
 # After build() has been called, the following are available:
 # - run_command:    command to be executed. E.g. ['/path/to/run'] or ['python3', '/path/to/main.py']. `None` if something failed.
@@ -111,6 +117,7 @@ def __init__(
         deps: Optional[list[Path]] = None,
         *,
         skip_double_build_warning=False,
+        limits: dict[str, int] = {},
     ):
         if deps is not None:
             assert isinstance(self, Generator)
@@ -149,6 +156,7 @@ def __init__(
         self.run_command: Optional[list[str]] = None
         self.hash: Optional[str] = None
         self.env: dict[str, int | str | Path] = {}
+        self.limits: dict[str, int] = limits
 
         self.ok = True
         self.built = False
@@ -275,7 +283,7 @@ def _get_language(self, bar: ProgressBar):
                 'mainclass': mainclass,
                 'Mainclass': mainclass[0].upper() + mainclass[1:],
                 # Memory limit in MB.
-                'memlim': (get_memory_limit() or 1024),
+                'memlim': self.limits.get('memory', 2048),
                 # Out-of-spec variables used by 'manual' and 'Viva' languages.
                 'build': (
                     self.tmpdir / 'build' if (self.tmpdir / 'build') in self.input_files else ''
@@ -372,10 +380,11 @@ def _compile(self, bar: ProgressBar):
             ret = exec_command(
                 self.compile_command,
                 stdout=subprocess.PIPE,
-                memory=5_000_000_000,
                 cwd=self.tmpdir,
                 # Compile errors are never cropped.
                 crop=False,
+                timeout=self.limits.get('compilation_time', None),
+                memory=self.limits.get('compilation_memory', None),
             )
         except FileNotFoundError as err:
             self.ok = False
@@ -478,8 +487,23 @@ def build(self, bar: ProgressBar):
         if self.path in self.problem._program_callbacks:
             for c in self.problem._program_callbacks[self.path]:
                 c(self)
+
+        if 'code' in self.limits:
+            size = sum(f.stat().st_size for f in self.source_files)
+            if size > self.limits['code'] * 1024:
+                bar.warn(
+                    f'Code limit exceeded (set limits.code to at least {(size + 1023) // 1024}KiB in problem.yaml)'
+                )
+
         return True
 
+    def _exec_command(self, *args, **kwargs):
+        if 'timeout' not in kwargs and 'timeout' in self.limits:
+            kwargs['timeout'] = self.limits['timeout']
+        if 'memory' not in kwargs and 'memory' in self.limits:
+            kwargs['memory'] = self.limits['memory']
+        return exec_command(*args, **kwargs)
+
     @staticmethod
     def add_callback(problem, path, c):
         if path not in problem._program_callbacks:
@@ -489,7 +513,9 @@ def add_callback(problem, path, c):
 
 class Generator(Program):
     def __init__(self, problem: "Problem", path: Path, **kwargs):
-        super().__init__(problem, path, 'generators', **kwargs)
+        super().__init__(
+            problem, path, 'generators', limits={'timeout': problem.limits.generator_time}, **kwargs
+        )
 
     # Run the generator in the given working directory.
     # May write files in |cwd| and stdout is piped to {name}.in if it's not written already.
@@ -509,11 +535,13 @@ def run(self, bar, cwd, name, args=[]):
             else:
                 f.unlink()
 
-        timeout = config.get_timeout()
+        timeout = self.limits['timeout']
 
         with stdout_path.open('w') as stdout_file:
-            result = exec_command(
-                self.run_command + args, stdout=stdout_file, timeout=timeout, cwd=cwd, memory=None
+            result = self._exec_command(
+                self.run_command + args,
+                stdout=stdout_file,
+                cwd=cwd,
             )
 
         result.retry = False
@@ -544,12 +572,19 @@ def run(self, bar, cwd, name, args=[]):
 
 class Visualizer(Program):
     def __init__(self, problem: "Problem", path: Path, **kwargs):
-        super().__init__(problem, path, 'visualizers', **kwargs)
+        super().__init__(
+            problem,
+            path,
+            'visualizers',
+            limits={'timeout': problem.limits.visualizer_time},
+            **kwargs,
+        )
 
     # Run the visualizer.
     # Stdin and stdout are not used.
     def run(self, cwd, args=[]):
         assert self.run_command is not None
-        return exec_command(
-            self.run_command + args, timeout=config.get_timeout(), cwd=cwd, memory=None
+        return self._exec_command(
+            self.run_command + args,
+            cwd=cwd,
         )