From a7ea5bad1a1d13d1c079a0d154e0d17a7a731a1d Mon Sep 17 00:00:00 2001 From: Georgios Kafanas Date: Sat, 3 Aug 2024 01:48:34 +0200 Subject: [PATCH 01/10] [feature] Copy build log and artifacts to a permanent location after failures The files can be build in some selected build path (--buildpath), and the logs of successful compilation are then concentrated to some other location for permanent storage (--logfile-format). Logs of failed builds remain in the build path location so that they can be inspected. However, this setup is problematic when building software in HPC jobs. Quite often in HPC systems the build path is set to some fast storage local to the node, like NVME raid mounted on `/tmp` or `/dev/shm` (as suggested in the documentation: https://docs.easybuild.io/configuration/#buildpath). The node storage is often wiped out after the end of a job, so the log files and the artifacts are no longer available after the termination of the job. This commit adds an option (--errorlogpath)to accumulate errors in some more permanent location, so that the can be easily inspected after a failed build. --- easybuild/framework/easyblock.py | 28 +++++++++++++++++++++++++--- easybuild/tools/config.py | 21 +++++++++++++++++++++ easybuild/tools/options.py | 8 +++++--- 3 files changed, 51 insertions(+), 6 deletions(-) diff --git a/easybuild/framework/easyblock.py b/easybuild/framework/easyblock.py index a515bc4c86..29d25e7bc1 100644 --- a/easybuild/framework/easyblock.py +++ b/easybuild/framework/easyblock.py @@ -75,13 +75,13 @@ from easybuild.tools.config import FORCE_DOWNLOAD_ALL, FORCE_DOWNLOAD_PATCHES, FORCE_DOWNLOAD_SOURCES from easybuild.tools.config import EASYBUILD_SOURCES_URL # noqa from easybuild.tools.config import build_option, build_path, get_log_filename, get_repository, get_repositorypath -from easybuild.tools.config import install_path, log_path, package_path, source_paths +from easybuild.tools.config import install_path, log_path, package_path, source_paths, error_log_path from easybuild.tools.environment import restore_env, sanitize_env from easybuild.tools.filetools import CHECKSUM_TYPE_MD5, CHECKSUM_TYPE_SHA256 from easybuild.tools.filetools import adjust_permissions, apply_patch, back_up_file, change_dir, check_lock -from easybuild.tools.filetools import compute_checksum, convert_name, copy_file, create_lock, create_patch_info +from easybuild.tools.filetools import convert_name, copy_file, copy_dir, create_lock, create_patch_info, is_readable from easybuild.tools.filetools import derive_alt_pypi_url, diff_files, dir_contains_files, download_file -from easybuild.tools.filetools import encode_class_name, extract_file +from easybuild.tools.filetools import encode_class_name, extract_file, compute_checksum from easybuild.tools.filetools import find_backup_name_candidate, get_source_tarball_from_git, is_alt_pypi_url from easybuild.tools.filetools import is_binary, is_sha256_checksum, mkdir, move_file, move_logs, read_file, remove_dir from easybuild.tools.filetools import remove_file, remove_lock, verify_checksum, weld_paths, write_file, symlink @@ -4460,6 +4460,28 @@ def ensure_writable_log_dir(log_dir): # there may be multiple log files, or the file name may be different due to zipping logs = glob.glob('%s*' % application_log) print_msg("Results of the build can be found in the log file(s) %s" % ', '.join(logs), log=_log, silent=silent) + err_log_path = error_log_path(ec=ecdict['ec']) + if err_log_path and not(success): + print_msg("Build log and artifacts copied to permanent storage: %s" % err_log_path, log=_log, silent=silent) + for log_file in logs: + target_file = os.path.join(err_log_path, os.path.basename(log_file)) + copy_file(log_file, target_file) + + name = ecdict['ec'].name + version = ecdict['ec'].version + + toolchain_dict = ecdict['ec'].toolchain.as_dict() + toolchain_components = [ + toolchain_dict['name'], + toolchain_dict['version'], + toolchain_dict['versionsuffix'], + ] + toolchain_components = [s for s in toolchain_components if len(s) > 0] + toolchain = '-'.join(toolchain_components) + + dest_build_path = os.path.join(err_log_path, name, version, toolchain) + if is_readable(app.builddir): + copy_dir(app.builddir, dest_build_path) del app diff --git a/easybuild/tools/config.py b/easybuild/tools/config.py index 2982c5eb2e..4ae161f47b 100644 --- a/easybuild/tools/config.py +++ b/easybuild/tools/config.py @@ -106,6 +106,7 @@ DEFAULT_PATH_SUBDIRS = { 'buildpath': 'build', 'containerpath': 'containers', + 'errorlogpath': 'error_log', 'installpath': '', 'packagepath': 'packages', 'repositorypath': 'ebfiles_repo', @@ -478,6 +479,7 @@ class ConfigurationVariables(BaseConfigurationVariables): 'buildpath', 'config', 'containerpath', + 'errorlogpath', 'installpath', 'installpath_modules', 'installpath_software', @@ -843,6 +845,25 @@ def log_path(ec=None): return log_file_format(return_directory=True, ec=ec, date=date, timestamp=timestamp) +def error_log_path(ec=None): + """ + Return the default error log path + + This is a path where file from the build_log_path can be stored permanently + :param ec: dict-like value that provides values for %(name)s and %(version)s template values + """ + error_log_path = ConfigurationVariables()['errorlogpath'] + + if ec is None: + ec = {} + + name, version = ec.get('name', '%(name)s'), ec.get('version', '%(version)s') + date = time.strftime("%Y%m%d") + timestamp = time.strftime("%H%M%S") + + return '/'.join([error_log_path, name + '-' + version, date + '-' + timestamp]) + + def get_build_log_path(): """ Return (temporary) directory for build log diff --git a/easybuild/tools/options.py b/easybuild/tools/options.py index 194e3a9969..7cd3592178 100644 --- a/easybuild/tools/options.py +++ b/easybuild/tools/options.py @@ -571,6 +571,8 @@ def config_options(self): 'envvars-user-modules': ("List of environment variables that hold the base paths for which user-specific " "modules will be installed relative to", 'strlist', 'store', [DEFAULT_ENVVAR_USERS_MODULES]), + 'errorlogpath': ("Location where logs and artifacts are copied in case of an error", + None, 'store', mk_full_default_path('errorlogpath')), 'external-modules-metadata': ("List of (glob patterns for) paths to files specifying metadata " "for external modules (INI format)", 'strlist', 'store', None), 'hooks': ("Location of Python module with hook implementations", 'str', 'store', None), @@ -1153,7 +1155,7 @@ def _postprocess_config(self): # - the could also specify the location of a *remote* (Git( repository, # which can be done in variety of formats (git@:/), https://, etc.) # (see also https://github.com/easybuilders/easybuild-framework/issues/3892); - path_opt_names = ['buildpath', 'containerpath', 'git_working_dirs_path', 'installpath', + path_opt_names = ['buildpath', 'containerpath', 'errorlogpath', 'git_working_dirs_path', 'installpath', 'installpath_modules', 'installpath_software', 'prefix', 'packagepath', 'robot_paths', 'sourcepath'] @@ -1163,8 +1165,8 @@ def _postprocess_config(self): if self.options.prefix is not None: # prefix applies to all paths, and repository has to be reinitialised to take new repositorypath in account # in the legacy-style configuration, repository is initialised in configuration file itself - path_opts = ['buildpath', 'containerpath', 'installpath', 'packagepath', 'repository', 'repositorypath', - 'sourcepath'] + path_opts = ['buildpath', 'containerpath', 'errorlogpath', 'installpath', 'packagepath', 'repository', + 'repositorypath', 'sourcepath'] for dest in path_opts: if not self.options._action_taken.get(dest, False): if dest == 'repository': From 43268a68fcd77c1c7798d205afef5355d54ac8aa Mon Sep 17 00:00:00 2001 From: Georgios Kafanas Date: Mon, 12 Aug 2024 21:58:58 +0200 Subject: [PATCH 02/10] Test error logging features Create tests for: - the `errorlogpath` option. --- test/framework/options.py | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/test/framework/options.py b/test/framework/options.py index 81620570fe..9363a795c1 100644 --- a/test/framework/options.py +++ b/test/framework/options.py @@ -5283,7 +5283,15 @@ def test_prefix_option(self): regex = re.compile(r"(?P\S*).*%s.*" % self.test_prefix, re.M) - expected = ['buildpath', 'containerpath', 'installpath', 'packagepath', 'prefix', 'repositorypath'] + expected = [ + 'buildpath', + 'containerpath', + 'errorlogpath', + 'installpath', + 'packagepath', + 'prefix', + 'repositorypath', + ] self.assertEqual(sorted(regex.findall(txt)), expected) def test_dump_env_script(self): From 7aa0819df80cc33ca020cd4b30ca4bae1dd9a22b Mon Sep 17 00:00:00 2001 From: Georgios Kafanas Date: Mon, 19 Aug 2024 15:02:48 +0200 Subject: [PATCH 03/10] [refactor] Extract artifact path from base build path and bilddir - There does not seem to be a field storing the path to the builddir of an easyblock relative to the base build path. In this refactored version the relative builddir is extracted from the full path and the base build path using the `os.path.relpath` function. - During the copying of the files, the operation may fail, for instance due to the lack of space in the target location or insufficient rights. Report the copying of the artifacts after the copy operations. --- easybuild/framework/easyblock.py | 23 +++++++++-------------- 1 file changed, 9 insertions(+), 14 deletions(-) diff --git a/easybuild/framework/easyblock.py b/easybuild/framework/easyblock.py index 29d25e7bc1..9540923fe1 100644 --- a/easybuild/framework/easyblock.py +++ b/easybuild/framework/easyblock.py @@ -4462,26 +4462,21 @@ def ensure_writable_log_dir(log_dir): print_msg("Results of the build can be found in the log file(s) %s" % ', '.join(logs), log=_log, silent=silent) err_log_path = error_log_path(ec=ecdict['ec']) if err_log_path and not(success): - print_msg("Build log and artifacts copied to permanent storage: %s" % err_log_path, log=_log, silent=silent) for log_file in logs: target_file = os.path.join(err_log_path, os.path.basename(log_file)) copy_file(log_file, target_file) - name = ecdict['ec'].name - version = ecdict['ec'].version + relative_build_artifact_log_path = os.path.relpath(app.builddir, build_path()) + build_artifact_log_path = os.path.join(err_log_path, relative_build_artifact_log_path) - toolchain_dict = ecdict['ec'].toolchain.as_dict() - toolchain_components = [ - toolchain_dict['name'], - toolchain_dict['version'], - toolchain_dict['versionsuffix'], - ] - toolchain_components = [s for s in toolchain_components if len(s) > 0] - toolchain = '-'.join(toolchain_components) - - dest_build_path = os.path.join(err_log_path, name, version, toolchain) if is_readable(app.builddir): - copy_dir(app.builddir, dest_build_path) + copy_dir(app.builddir, build_artifact_log_path) + + print_msg( + "Build log and any output artifacts copied to permanent storage: %s" % err_log_path, + log=_log, + silent=silent + ) del app From 0a2ef64b7e45c8b6279fd8dc68a446b47bde9e25 Mon Sep 17 00:00:00 2001 From: Georgios Kafanas Date: Mon, 19 Aug 2024 16:12:06 +0200 Subject: [PATCH 04/10] [refactor] Extract function persisting logs and artifacts The function moves logs and artifacts of failed build in a special location for permanent storage. --- easybuild/framework/easyblock.py | 46 +++++++++++++++++--------------- 1 file changed, 25 insertions(+), 21 deletions(-) diff --git a/easybuild/framework/easyblock.py b/easybuild/framework/easyblock.py index 9540923fe1..3a58d5816f 100644 --- a/easybuild/framework/easyblock.py +++ b/easybuild/framework/easyblock.py @@ -4198,6 +4198,29 @@ def print_dry_run_note(loc, silent=True): dry_run_msg(msg, silent=silent) +def persists_failed_compilation_log_and_artifacts(success, application_log, log, silent, builddir, err_log_path): + if application_log: + # there may be multiple log files, or the file name may be different due to zipping + logs = glob.glob('%s*' % application_log) + print_msg("Results of the build can be found in the log file(s) %s" % ', '.join(logs), log=log, silent=silent) + if err_log_path and not(success): + for log_file in logs: + target_file = os.path.join(err_log_path, os.path.basename(log_file)) + copy_file(log_file, target_file) + + relative_build_artifact_log_path = os.path.relpath(builddir, build_path()) + build_artifact_log_path = os.path.join(err_log_path, relative_build_artifact_log_path) + + if is_readable(builddir): + copy_dir(builddir, build_artifact_log_path) + + print_msg( + "Build log and any output artifacts copied to permanent storage: %s" % err_log_path, + log=_log, + silent=silent + ) + + def build_and_install_one(ecdict, init_env): """ Build the software @@ -4456,27 +4479,8 @@ def ensure_writable_log_dir(log_dir): else: dry_run_msg("(no ignored errors during dry run)\n", silent=silent) - if application_log: - # there may be multiple log files, or the file name may be different due to zipping - logs = glob.glob('%s*' % application_log) - print_msg("Results of the build can be found in the log file(s) %s" % ', '.join(logs), log=_log, silent=silent) - err_log_path = error_log_path(ec=ecdict['ec']) - if err_log_path and not(success): - for log_file in logs: - target_file = os.path.join(err_log_path, os.path.basename(log_file)) - copy_file(log_file, target_file) - - relative_build_artifact_log_path = os.path.relpath(app.builddir, build_path()) - build_artifact_log_path = os.path.join(err_log_path, relative_build_artifact_log_path) - - if is_readable(app.builddir): - copy_dir(app.builddir, build_artifact_log_path) - - print_msg( - "Build log and any output artifacts copied to permanent storage: %s" % err_log_path, - log=_log, - silent=silent - ) + err_log_path = error_log_path(ec=ecdict['ec']) + persists_failed_compilation_log_and_artifacts(success, application_log, _log, silent, app.builddir, err_log_path) del app From f7666ae3a85da11227ac13ab1e8ac6e6460be9fc Mon Sep 17 00:00:00 2001 From: Georgios Kafanas Date: Fri, 23 Aug 2024 09:36:44 +0200 Subject: [PATCH 05/10] [refactor] Extract function generating the relative base builddir path The base builddir path is used to construct the builddir by - pre-pending the asboloute build path, and - adding a numerical suffix to ensure uniqueness. --- easybuild/framework/easyblock.py | 29 ++++++++++++++++++++--------- 1 file changed, 20 insertions(+), 9 deletions(-) diff --git a/easybuild/framework/easyblock.py b/easybuild/framework/easyblock.py index 3a58d5816f..d563fbd5fa 100644 --- a/easybuild/framework/easyblock.py +++ b/easybuild/framework/easyblock.py @@ -1045,8 +1045,8 @@ def moduleGenerator(self): # # DIRECTORY UTILITY FUNCTIONS # - def gen_builddir(self): - """Generate the (unique) name for the builddir""" + def get_relative_builddir_base_path(self): + """Generate builddir base name relative to build_path""" clean_name = remove_unwanted_chars(self.name) # if a toolchain version starts with a -, remove the - so prevent a -- in the path name @@ -1054,7 +1054,14 @@ def gen_builddir(self): tcversion = tc['version'].lstrip('-') lastdir = "%s%s-%s%s" % (self.cfg['versionprefix'], tc['name'], tcversion, self.cfg['versionsuffix']) - builddir = os.path.join(os.path.abspath(build_path()), clean_name, self.version, lastdir) + relative_builddir = os.path.join(clean_name, self.version, lastdir) + + return relative_builddir + + def gen_builddir(self): + """Generate the (unique) name for the builddir""" + relative_builddir = self.get_relative_builddir_base_path() + builddir = os.path.join(os.path.abspath(build_path()), relative_builddir) # make sure build dir is unique if cleanupoldbuild is False or not set if not self.cfg.get('cleanupoldbuild', False): @@ -4198,20 +4205,24 @@ def print_dry_run_note(loc, silent=True): dry_run_msg(msg, silent=silent) -def persists_failed_compilation_log_and_artifacts(success, application_log, log, silent, builddir, err_log_path): +def persists_failed_compilation_log_and_artifacts(success, application_log, silent, app, err_log_path): if application_log: # there may be multiple log files, or the file name may be different due to zipping logs = glob.glob('%s*' % application_log) - print_msg("Results of the build can be found in the log file(s) %s" % ', '.join(logs), log=log, silent=silent) + print_msg( + "Results of the build can be found in the log file(s) %s" % ', '.join(logs), + log=_log, + silent=silent + ) + if err_log_path and not(success): for log_file in logs: target_file = os.path.join(err_log_path, os.path.basename(log_file)) copy_file(log_file, target_file) - relative_build_artifact_log_path = os.path.relpath(builddir, build_path()) - build_artifact_log_path = os.path.join(err_log_path, relative_build_artifact_log_path) - + builddir = app.builddir if is_readable(builddir): + build_artifact_log_path = os.path.join(err_log_path, app.get_relative_builddir_base_path()) copy_dir(builddir, build_artifact_log_path) print_msg( @@ -4480,7 +4491,7 @@ def ensure_writable_log_dir(log_dir): dry_run_msg("(no ignored errors during dry run)\n", silent=silent) err_log_path = error_log_path(ec=ecdict['ec']) - persists_failed_compilation_log_and_artifacts(success, application_log, _log, silent, app.builddir, err_log_path) + persists_failed_compilation_log_and_artifacts(success, application_log, silent, app, err_log_path) del app From 7e162f628b9b75d0c42d3203f10cf958a11acc07 Mon Sep 17 00:00:00 2001 From: Georgios Kafanas Date: Mon, 9 Sep 2024 00:35:38 +0200 Subject: [PATCH 06/10] [style] Make a distinction between temporary and permanent log files The log messages mention both the temporary log file created in the build directory, and the path where the file is copied for permanent storage. This commits makes a distinction between the two path in the log messages. --- easybuild/framework/easyblock.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/easybuild/framework/easyblock.py b/easybuild/framework/easyblock.py index d563fbd5fa..c7f9dd3929 100644 --- a/easybuild/framework/easyblock.py +++ b/easybuild/framework/easyblock.py @@ -4210,7 +4210,7 @@ def persists_failed_compilation_log_and_artifacts(success, application_log, sile # there may be multiple log files, or the file name may be different due to zipping logs = glob.glob('%s*' % application_log) print_msg( - "Results of the build can be found in the log file(s) %s" % ', '.join(logs), + "Results of the build can be found in the temporary log file(s) %s" % ', '.join(logs), log=_log, silent=silent ) From be4f960fbbcf5087bcce3c1524d47fe7a6b471b9 Mon Sep 17 00:00:00 2001 From: Kenneth Hoste Date: Wed, 11 Sep 2024 08:06:59 +0200 Subject: [PATCH 07/10] fix minor code style issue by using "not success" instead of "not(success)" --- easybuild/framework/easyblock.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/easybuild/framework/easyblock.py b/easybuild/framework/easyblock.py index c7f9dd3929..e728917d8e 100644 --- a/easybuild/framework/easyblock.py +++ b/easybuild/framework/easyblock.py @@ -4215,7 +4215,7 @@ def persists_failed_compilation_log_and_artifacts(success, application_log, sile silent=silent ) - if err_log_path and not(success): + if err_log_path and not success: for log_file in logs: target_file = os.path.join(err_log_path, os.path.basename(log_file)) copy_file(log_file, target_file) From 469801aef18f1eba5b40c1bf5aff5e1848b95985 Mon Sep 17 00:00:00 2001 From: Georgios Kafanas Date: Wed, 11 Sep 2024 09:31:45 +0200 Subject: [PATCH 08/10] [style] Use more expressive input variable name --- easybuild/framework/easyblock.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/easybuild/framework/easyblock.py b/easybuild/framework/easyblock.py index e728917d8e..ff425d4bfb 100644 --- a/easybuild/framework/easyblock.py +++ b/easybuild/framework/easyblock.py @@ -4205,7 +4205,7 @@ def print_dry_run_note(loc, silent=True): dry_run_msg(msg, silent=silent) -def persists_failed_compilation_log_and_artifacts(success, application_log, silent, app, err_log_path): +def persists_failed_compilation_log_and_artifacts(build_successful, application_log, silent, app, err_log_path): if application_log: # there may be multiple log files, or the file name may be different due to zipping logs = glob.glob('%s*' % application_log) @@ -4215,7 +4215,7 @@ def persists_failed_compilation_log_and_artifacts(success, application_log, sile silent=silent ) - if err_log_path and not success: + if err_log_path and not build_successful: for log_file in logs: target_file = os.path.join(err_log_path, os.path.basename(log_file)) copy_file(log_file, target_file) From 90565c3446a0e30cba5c9ffee666417dcee26b6d Mon Sep 17 00:00:00 2001 From: Georgios Kafanas Date: Wed, 11 Sep 2024 12:17:23 +0200 Subject: [PATCH 09/10] [bugfix] Cover edge cases with indistinguishable error log paths - In testing multiple failures can occur in quick succession resulting in the same time stamp, and as a result in the same base error log path. Extent the path stamp with an increasing number (naive O(n^2) algorithm used at the moment, should be sufficient). - In case the user provides the same error log path as the build directory log path, add a check to prevent copying the files to prevent errors in the copying functions. --- easybuild/framework/easyblock.py | 11 +++++++++-- easybuild/tools/config.py | 10 +++++++++- 2 files changed, 18 insertions(+), 3 deletions(-) diff --git a/easybuild/framework/easyblock.py b/easybuild/framework/easyblock.py index ff425d4bfb..2bb1c5282b 100644 --- a/easybuild/framework/easyblock.py +++ b/easybuild/framework/easyblock.py @@ -4206,6 +4206,13 @@ def print_dry_run_note(loc, silent=True): def persists_failed_compilation_log_and_artifacts(build_successful, application_log, silent, app, err_log_path): + def do_if_paths_distinct(operation, source, destination): + if not os.path.exists(source): + return + if os.path.realpath(source) == os.path.realpath(destination): + return + operation(source, destination) + if application_log: # there may be multiple log files, or the file name may be different due to zipping logs = glob.glob('%s*' % application_log) @@ -4218,12 +4225,12 @@ def persists_failed_compilation_log_and_artifacts(build_successful, application_ if err_log_path and not build_successful: for log_file in logs: target_file = os.path.join(err_log_path, os.path.basename(log_file)) - copy_file(log_file, target_file) + do_if_paths_distinct(copy_file, log_file, target_file) builddir = app.builddir if is_readable(builddir): build_artifact_log_path = os.path.join(err_log_path, app.get_relative_builddir_base_path()) - copy_dir(builddir, build_artifact_log_path) + do_if_paths_distinct(copy_dir, builddir, build_artifact_log_path) print_msg( "Build log and any output artifacts copied to permanent storage: %s" % err_log_path, diff --git a/easybuild/tools/config.py b/easybuild/tools/config.py index 4ae161f47b..9fea9999c8 100644 --- a/easybuild/tools/config.py +++ b/easybuild/tools/config.py @@ -861,7 +861,15 @@ def error_log_path(ec=None): date = time.strftime("%Y%m%d") timestamp = time.strftime("%H%M%S") - return '/'.join([error_log_path, name + '-' + version, date + '-' + timestamp]) + base_path = '/'.join([error_log_path, name + '-' + version, date + '-' + timestamp]) + + path = base_path + inc_no = 1 + while os.path.exists(path): + path = base_path + '_' + str(inc_no) + inc_no += 1 + + return path def get_build_log_path(): From 0eb8e61e714052006656dbab2db3dea2e26e24f0 Mon Sep 17 00:00:00 2001 From: Georgios Kafanas Date: Sat, 21 Dec 2024 20:43:18 +0100 Subject: [PATCH 10/10] Add test for error logging The toy test file is modified with a patch to fail during compilation. The tests verify that: - the source directory is copied to the error log path, - the log files are copied to the error log path, and - a warning for the compilation failure is reported in stdout. --- test/framework/docs.py | 2 + .../test_ecs/t/toy/toy-0.0-buggy.eb | 33 ++++++++ .../test_ecs/t/toy/toy-0.0-test.eb | 2 +- test/framework/filetools.py | 2 +- .../sandbox/sources/toy/toy-0.0_add-bug.patch | 10 +++ test/framework/toy_build.py | 83 ++++++++++++++++++- test/framework/utilities.py | 27 ++++++ 7 files changed, 153 insertions(+), 6 deletions(-) create mode 100644 test/framework/easyconfigs/test_ecs/t/toy/toy-0.0-buggy.eb create mode 100644 test/framework/sandbox/sources/toy/toy-0.0_add-bug.patch diff --git a/test/framework/docs.py b/test/framework/docs.py index 93db1ad505..509bca7d08 100644 --- a/test/framework/docs.py +++ b/test/framework/docs.py @@ -724,6 +724,7 @@ def test_list_software(self): 'homepage: https://easybuilders.github.io/easybuild', '', " * toy v0.0: gompi/2018a, system", + " * toy v0.0 (versionsuffix: '-buggy'): system", " * toy v0.0 (versionsuffix: '-deps'): system", " * toy v0.0 (versionsuffix: '-iter'): system", " * toy v0.0 (versionsuffix: '-multiple'): system", @@ -746,6 +747,7 @@ def test_list_software(self): 'version versionsuffix toolchain', '======= ============= ===========================', '``0.0`` ``gompi/2018a``, ``system``', + '``0.0`` ``-buggy`` ``system``', '``0.0`` ``-deps`` ``system``', '``0.0`` ``-iter`` ``system``', '``0.0`` ``-multiple`` ``system``', diff --git a/test/framework/easyconfigs/test_ecs/t/toy/toy-0.0-buggy.eb b/test/framework/easyconfigs/test_ecs/t/toy/toy-0.0-buggy.eb new file mode 100644 index 0000000000..89fe5979b6 --- /dev/null +++ b/test/framework/easyconfigs/test_ecs/t/toy/toy-0.0-buggy.eb @@ -0,0 +1,33 @@ +name = 'toy' +version = '0.0' +versionsuffix = '-buggy' + +homepage = 'https://easybuilders.github.io/easybuild' +description = "Toy C program, 100% toy." + +toolchain = SYSTEM + +sources = [SOURCE_TAR_GZ] +checksums = [[ + 'be662daa971a640e40be5c804d9d7d10', # default (MD5) + '44332000aa33b99ad1e00cbd1a7da769220d74647060a10e807b916d73ea27bc', # default (SHA256) + ('adler32', '0x998410035'), + ('crc32', '0x1553842328'), + ('md5', 'be662daa971a640e40be5c804d9d7d10'), + ('sha1', 'f618096c52244539d0e89867405f573fdb0b55b0'), + ('size', 273), +]] +patches = [ + 'toy-0.0_add-bug.patch', + ('toy-extra.txt', 'toy-0.0'), +] + +sanity_check_paths = { + 'files': [('bin/yot', 'bin/toy')], + 'dirs': ['bin'], +} + +postinstallcmds = ["echo TOY > %(installdir)s/README"] + +moduleclass = 'tools' +# trailing comment, leave this here, it may trigger bugs with extract_comments() diff --git a/test/framework/easyconfigs/test_ecs/t/toy/toy-0.0-test.eb b/test/framework/easyconfigs/test_ecs/t/toy/toy-0.0-test.eb index 90cc7429d3..9184a55d23 100644 --- a/test/framework/easyconfigs/test_ecs/t/toy/toy-0.0-test.eb +++ b/test/framework/easyconfigs/test_ecs/t/toy/toy-0.0-test.eb @@ -18,7 +18,7 @@ checksums = [[ ('size', 273), ]] patches = [ - 'toy-0.0_fix-silly-typo-in-printf-statement.patch', + 'toy-0.0_add-bug.patch', ('toy-extra.txt', 'toy-0.0'), ] diff --git a/test/framework/filetools.py b/test/framework/filetools.py index 2e2b030fc8..03197de67d 100644 --- a/test/framework/filetools.py +++ b/test/framework/filetools.py @@ -2474,7 +2474,7 @@ def test_index_functions(self): # test with specified path with and without trailing '/'s for path in [test_ecs, test_ecs + '/', test_ecs + '//']: index = ft.create_index(path) - self.assertEqual(len(index), 94) + self.assertEqual(len(index), 95) expected = [ os.path.join('b', 'bzip2', 'bzip2-1.0.6-GCC-4.9.2.eb'), diff --git a/test/framework/sandbox/sources/toy/toy-0.0_add-bug.patch b/test/framework/sandbox/sources/toy/toy-0.0_add-bug.patch new file mode 100644 index 0000000000..7512447168 --- /dev/null +++ b/test/framework/sandbox/sources/toy/toy-0.0_add-bug.patch @@ -0,0 +1,10 @@ +--- a/toy-0.0.orig/toy.source 2014-03-06 18:48:16.000000000 +0100 ++++ b/toy-0.0/toy.source 2020-08-18 12:19:35.000000000 +0200 +@@ -2,6 +2,6 @@ + + int main(int argc, char* argv[]){ + +- printf("I'm a toy, and proud of it.\n"); ++ printf("I'm a toy, and proud of it.\n") + return 0; + } diff --git a/test/framework/toy_build.py b/test/framework/toy_build.py index 5219ee5aa6..8c67134b2e 100644 --- a/test/framework/toy_build.py +++ b/test/framework/toy_build.py @@ -40,8 +40,10 @@ import sys import tempfile import textwrap +import pathlib +import filecmp from easybuild.tools import LooseVersion -from test.framework.utilities import EnhancedTestCase, TestLoaderFiltered +from test.framework.utilities import EnhancedTestCase, TestLoaderFiltered, TempDirectory, conditional_regexs_present from test.framework.package import mock_fpm from unittest import TextTestRunner @@ -156,9 +158,9 @@ def check_toy(self, installpath, outtxt, name='toy', version='0.0', versionprefi devel_module_path = os.path.join(software_path, 'easybuild', '%s-%s-easybuild-devel' % (name, full_version)) self.assertExists(devel_module_path) - def test_toy_build(self, extra_args=None, ec_file=None, tmpdir=None, verify=True, fails=False, verbose=True, - raise_error=False, test_report=None, name='toy', versionsuffix='', testing=True, - raise_systemexit=False, force=True, test_report_regexs=None, debug=True): + def test_toy_build(self, extra_args=None, ec_file=None, tmpdir=None, tmp_logdir=None, verify=True, fails=False, + verbose=True, raise_error=False, test_report=None, name='toy', versionsuffix='', testing=True, + raise_systemexit=False, force=True, test_report_regexs=None, debug=True, check_errorlog=None): """Perform a toy build.""" if extra_args is None: extra_args = [] @@ -172,6 +174,7 @@ def test_toy_build(self, extra_args=None, ec_file=None, tmpdir=None, verify=True '--sourcepath=%s' % self.test_sourcepath, '--buildpath=%s' % self.test_buildpath, '--installpath=%s' % self.test_installpath, + '--errorlogpath=%s' % self.test_errorlogpath, '--unittest-file=%s' % self.logfile, '--robot=%s' % os.pathsep.join([self.test_buildpath, os.path.dirname(__file__)]), ] @@ -183,6 +186,8 @@ def test_toy_build(self, extra_args=None, ec_file=None, tmpdir=None, verify=True args.append('--tmpdir=%s' % tmpdir) if test_report is not None: args.append('--dump-test-report=%s' % test_report) + if tmp_logdir is not None: + args.append('--tmp-logdir=%s' % tmp_logdir) args.extend(extra_args) myerr = None try: @@ -228,6 +233,9 @@ def test_toy_build(self, extra_args=None, ec_file=None, tmpdir=None, verify=True msg = "Pattern %s found in full test report: %s" % (regex.pattern, test_report_txt) self.assertTrue(regex.search(test_report_txt), msg) + if check_errorlog is not None: + check_errorlog(outtxt, tmp_logdir, self.test_buildpath, self.test_errorlogpath) + return outtxt def run_test_toy_build_with_output(self, *args, **kwargs): @@ -271,6 +279,73 @@ def test_toy_broken(self): # cleanup shutil.rmtree(tmpdir) + def detect_log_file(self, tmp_logpath): + log_files = list(pathlib.Path(tmp_logpath).glob("**/*.log")) + + self.assertTrue(len(log_files) >= 1, 'Log files generated') + log_file = log_files[0] + self.assertTrue(len(log_files) == 1, f"Log file '{log_file}' is unique") + + return log_file + + def assert_build_files_copied(self, buildpath, errorlogpath): + buildir = pathlib.Path(buildpath) + errorlogdir = pathlib.Path(errorlogpath) + iso_date_pattern = r'????????-??????' + for file in buildir.iterdir(): + file_relative_path = file.relative_to(buildir) + file_copies = list(errorlogdir.glob(f"toy-0.0/{iso_date_pattern}/{file_relative_path}")) + self.assertTrue(len(file_copies) == 1, f"Unique copy of toy build file '{file}' made") + file_copy = file_copies[0] + + if file_copy.is_file(): + msg = f"File '{file}' copied succesfully" + self.assertTrue(filecmp.cmp(str(file), str(file_copy), shallow=False), msg) + + def assert_log_files_copied(self, log_file, errorlogpath): + file_name = log_file.name + saved_log_files = list(pathlib.Path(errorlogpath).glob(f"**/{file_name}")) + self.assertTrue(len(saved_log_files) == 1, f"Unique copy of log file '{log_file}' made") + for saved_log_file in saved_log_files: + msg = f"Log file '{log_file}' copied succesfully" + self.assertTrue(filecmp.cmp(str(log_file), str(saved_log_file), shallow=False), msg) + + def assert_error_reported(self, outtxt, conditional_regexs, output_regexs): + if conditional_regexs_present(conditional_regexs, outtxt): + for regex_pattern in output_regexs: + regex = re.compile(regex_pattern, re.M) + msg_stdout = "Pattern %s found in full test report: %s" % (regex.pattern, outtxt) + self.assertTrue(regex.search(outtxt), msg_stdout) + + def check_errorlog(self, conditional_regexs, output_regexs, outtxt, tmp_logpath, buildpath, errorlogpath): + log_file = self.detect_log_file(tmp_logpath) + + self.assert_build_files_copied(buildpath, errorlogpath) + self.assert_log_files_copied(log_file, errorlogpath) + self.assert_error_reported(outtxt, conditional_regexs, output_regexs) + + with open(f"{log_file}", 'r') as p_log_file: + self.assert_error_reported(p_log_file.read(), conditional_regexs, output_regexs) + + def test_toy_broken_compilation(self): + """Test whether log files and the build directory are copied to a permanent location after a failed + compilation.""" + tmpdir = TempDirectory() + tmp_logdir = TempDirectory() + broken_compilation_ec = os.path.join(os.path.dirname(__file__), + 'easyconfigs', 'test_ecs', 't', 'toy', 'toy-0.0-buggy.eb') + + conditional_regexs = [r'gcc toy\.c -o toy'] # Cover environments where compilation not attempted + output_regexs = [r"^\s+toy\.c:5:44: error: expected (;|';') before"] + + def check_errorlog(outtxt, tmp_logpath, buildpath, errorlogpath): + self.check_errorlog(conditional_regexs, output_regexs, outtxt, tmp_logpath, buildpath, errorlogpath) + + self.run_test_toy_build_with_output( + ec_file=broken_compilation_ec, tmpdir=tmpdir.get_path(), tmp_logdir=tmp_logdir.get_path(), + verify=False, fails=True, verbose=False, raise_error=False, + name='toy', versionsuffix='-buggy', check_errorlog=check_errorlog) + def test_toy_tweaked(self): """Test toy build with tweaked easyconfig, for testing extra easyconfig parameters.""" test_ecs_dir = os.path.join(os.path.abspath(os.path.dirname(__file__)), 'easyconfigs') diff --git a/test/framework/utilities.py b/test/framework/utilities.py index bf44ad828d..f2f1122a94 100644 --- a/test/framework/utilities.py +++ b/test/framework/utilities.py @@ -121,6 +121,8 @@ def setUp(self): os.environ['EASYBUILD_BUILDPATH'] = self.test_buildpath self.test_installpath = tempfile.mkdtemp() os.environ['EASYBUILD_INSTALLPATH'] = self.test_installpath + self.test_errorlogpath = tempfile.mkdtemp() + os.environ['EASYBUILD_ERRORLOGPATH'] = self.test_errorlogpath # make sure that the tests only pick up easyconfigs provided with the tests os.environ['EASYBUILD_ROBOT_PATHS'] = os.path.join(testdir, 'easyconfigs', 'test_ecs') @@ -514,3 +516,28 @@ def find_full_path(base_path, trim=(lambda x: x)): break return full_path + + +def conditional_regexs_present(conditional_regexs, outtxt): + if conditional_regexs == []: + return True + + for conditional_regex in conditional_regexs: + regex = re.compile(conditional_regex, re.M) + found = regex.search(outtxt) + if found: + return True + + return False + + +class TempDirectory: + def __init__(self): + self.dir = tempfile.mkdtemp() + + def __del__(self): + if os.path.exists(self.dir): + shutil.rmtree(self.dir) + + def get_path(self): + return self.dir