diff --git a/scripts/update-dashboard.sh b/scripts/update-dashboard.sh index 292d987..10cbe5b 100755 --- a/scripts/update-dashboard.sh +++ b/scripts/update-dashboard.sh @@ -1,12 +1,16 @@ source $HOME/.ci-auth skare3-dashboard -o /proj/sot/ska/jgonzalez/index.html +skare3-dashboard -o /proj/sot/ska/jgonzalez/packages.json skare3-test-dashboard -o /proj/sot/ska/jgonzalez/test_results.html --static-dir https://cxc.cfa.harvard.edu/mta/ASPECT/skare3/dashboard/static --log-dir https://icxc.cfa.harvard.edu/aspect/skare3/dashboard/tests +skare3-test-dashboard -o /proj/sot/ska/jgonzalez/test_results.json # horrible hack: rm -f /proj/sot/ska/www/ASPECT_ICXC/skare3/dashboard/tests ln -s `/proj/sot/ska/jgonzalez/git/skare3_tools/scripts/test_dir.py` /proj/sot/ska/www/ASPECT_ICXC/skare3/dashboard/tests mv /proj/sot/ska/jgonzalez/index.html /proj/sot/ska/www/ASPECT/skare3/dashboard/ +mv /proj/sot/ska/jgonzalez/packages.json /proj/sot/ska/www/ASPECT/skare3/dashboard/ mv /proj/sot/ska/jgonzalez/test_results.html /proj/sot/ska/www/ASPECT/skare3/dashboard/tests/index.html +mv /proj/sot/ska/jgonzalez/test_results.json /proj/sot/ska/www/ASPECT/skare3/dashboard/tests/ # echo files copied to /proj/sot/ska/www/ASPECT/skare3/dashboard # echo dashboard at https://cxc.cfa.harvard.edu/mta/ASPECT/skare3/dashboard/ diff --git a/skare3_tools/dashboard/views/dashboard.py b/skare3_tools/dashboard/views/dashboard.py index d8abc34..785ff0e 100755 --- a/skare3_tools/dashboard/views/dashboard.py +++ b/skare3_tools/dashboard/views/dashboard.py @@ -2,6 +2,8 @@ import argparse import datetime +import json +from pathlib import Path from skare3_tools import packages from skare3_tools import test_results as tr @@ -10,7 +12,7 @@ package_name_map = packages.get_package_list() -def dashboard(config=None): +def dashboard(config=None, render=True): if config is None: config = {"static_dir": "static"} @@ -60,6 +62,9 @@ def dashboard(config=None): else: p["test_status"] = "PASS" + if not render: + return info + template = get_template("dashboard.html") return template.render(title="Skare3 Packages", info=info, config=config) @@ -73,6 +78,7 @@ def get_parser(): metavar="FILENAME", help="Output file (default: index.html)", default="index.html", + type=Path, ) return parser @@ -80,7 +86,10 @@ def get_parser(): def main(): args = get_parser().parse_args() with open(args.o, "w") as out: - out.write(dashboard()) + if args.o.suffix == ".json": + json.dump(dashboard(render=False), out) + else: + out.write(dashboard()) if __name__ == "__main__": diff --git a/skare3_tools/dashboard/views/test_results.py b/skare3_tools/dashboard/views/test_results.py index 65c35b6..0bcb972 100755 --- a/skare3_tools/dashboard/views/test_results.py +++ b/skare3_tools/dashboard/views/test_results.py @@ -2,8 +2,8 @@ import argparse import json -import os import webbrowser +from pathlib import Path from skare3_tools import dashboard from skare3_tools import test_results as tr @@ -15,10 +15,10 @@ def test_results(): "static_dir": "static", "log_dir": "tests/logs/{uid}".format(uid=test_run["run_info"]["uid"]), } - return _render(test_run, config) + return _get_results(test_run, config) -def _render(tests, config): +def _get_results(tests, config, render=True): if "run_info" not in tests: tests["run_info"] = { k: "" @@ -56,6 +56,9 @@ def _render(tests, config): ts["pass"] = n_pass ts["fail"] = n_fail + if not render: + return tests + template = dashboard.get_template("test-results.html") return template.render(title="Skare3 Tests", data=tests, config=config) @@ -69,6 +72,7 @@ def get_parser(): help="Directory or JSON file containing all test results. " "If it is a directory, then it must have a file named all_tests.json.", dest="file_in", + type=Path, ) parser.add_argument( "-o", @@ -76,6 +80,7 @@ def get_parser(): "index.html, located in the input directory or the current " "working directory, depending on whether the '-i' options was given.", dest="file_out", + type=Path, ) parser.add_argument( "-b", @@ -96,16 +101,16 @@ def main(): parser = get_parser() args = parser.parse_args() config = {"static_dir": args.static_dir, "log_dir": args.log_dir} - if args.file_in and os.path.isdir(args.file_in): - args.file_in = os.path.join(args.file_in, "all_tests.json") + if args.file_in and args.file_in.is_dir(): + args.file_in = args.file_in / "all_tests.json" if args.file_out is None: if args.file_in: - args.file_out = os.path.join(os.path.dirname(args.file_in), "index.html") + args.file_out = args.file_in.parent / "index.html" else: args.file_out = "index.html" - if args.file_in and not os.path.exists(args.file_in): + if args.file_in and not args.file_in.exists(): print("{filename} does not exist".format(filename=args.file_in)) parser.print_help() parser.exit(1) @@ -120,10 +125,13 @@ def main(): results["run_info"][key] = ", ".join(results["run_info"][key]) with open(args.file_out, "w") as out: - out.write(_render(results, config)) + if args.file_out.suffix == ".json": + json.dump(_get_results(results, config, render=False), out) + else: + out.write(_get_results(results, config)) - if not args.b: - file_out = os.path.abspath(args.file_out) + if not args.b and args.file_out.suffix in [".html", ".htm"]: + file_out = args.file_out.absolute() webbrowser.open("file://{file_out}".format(file_out=file_out), new=2) diff --git a/skare3_tools/packages.py b/skare3_tools/packages.py index 4727ecb..4a2354f 100755 --- a/skare3_tools/packages.py +++ b/skare3_tools/packages.py @@ -433,7 +433,6 @@ def _get_repository_info_v4( for r in releases: r["tag_oid"], r["committed_date"] = _get_tag_target(r["tag"]) - releases = {r["tag_oid"]: r for r in releases} release_info = [ { "release_tag": "", @@ -466,17 +465,20 @@ def _get_repository_info_v4( for commit in commits: sha = commit["oid"] - if sha in releases: - release_info.append( - { - "release_sha": sha, - "release_commit_date": releases[sha]["committed_date"], - "release_tag": releases[sha]["tagName"], - "release_tag_date": releases[sha]["publishedAt"], - "commits": [], - "merges": [], - } - ) + releases_at_commit = [ + { + "release_sha": release["tag_oid"], + "release_commit_date": release["committed_date"], + "release_tag": release["tagName"], + "release_tag_date": release["publishedAt"], + "commits": [], + "merges": [], + } + for release in releases + if release["tag_oid"] == sha + ] + release_info += releases_at_commit + release_info[-1]["commits"].append(commit) match = re.match( r"Merge pull request #(?P.+) from (?P\S+)\n\n(?P.+)", @@ -504,14 +506,14 @@ def _get_repository_info_v4( # Now we will add the remaining releases, which presumably happened in another branch. release_shas = [r["release_sha"] for r in release_info[1:]] - for sha in releases: - if sha not in release_shas: + for release in releases: + if release["tag_oid"] not in release_shas: release_info.append( { - "release_sha": sha, - "release_tag": releases[sha]["tagName"], - "release_tag_date": releases[sha]["publishedAt"], - "release_commit_date": releases[sha]["committed_date"], + "release_sha": release["tag_oid"], + "release_tag": release["tagName"], + "release_tag_date": release["publishedAt"], + "release_commit_date": release["committed_date"], "commits": [], "merges": [], } @@ -661,12 +663,15 @@ def _split_versions(depends): """ result = {} for depend in depends: - v = depend.split("==") if "==" in depend else depend.split() - if len(v) > 2: - raise Exception(f"Version spec got split into too many parts: {depend}") - p_name = v[0].strip() - p_version = v[1].strip() if len(v) == 2 else "" - result[p_name] = p_version + if "==" in depend: + name_version = depend.split("==", maxsplit=1) + else: + name_version = depend.split(maxsplit=1) + if len(name_version) == 2: + name, version = name_version + else: + name, version = name_version[0], "" + result[name.strip()] = version.strip() return result @@ -748,9 +753,6 @@ def _get_repository_info_v3( for r, c in zip(releases, release_commits) } - # later on, the releases are referred by commit sha - releases = {c["sha"]: r for r, c in zip(releases, release_commits)} - date_since = None if type(since) is int: # only the latest 'since' releases (at most) will be included in summary @@ -780,15 +782,18 @@ def _get_repository_info_v3( commits = commits[:-1] # remove first commit, which was just the starting point for commit in commits: sha = commit["sha"] - if sha in releases.keys(): - release_info.append( - { - "release_tag": releases[sha]["tag_name"], - "release_tag_date": releases[sha]["published_at"], - "commits": [], - "merges": [], - } - ) + releases_at_commit = [ + { + "release_tag": release["tag_name"], + "release_tag_date": release["published_at"], + "commits": [], + "merges": [], + } + for release in [ + r for r, c in zip(releases, release_commits) if c["sha"] == sha + ] + ] + release_info += releases_at_commit release_info[-1]["commits"].append( {