Merge pull request #94 from sot/dashboard

Fixes and improvements in Ska dashboard

scripts/update-dashboard.sh

@@ -1,12 +1,16 @@
 source $HOME/.ci-auth
 skare3-dashboard -o /proj/sot/ska/jgonzalez/index.html
+skare3-dashboard -o /proj/sot/ska/jgonzalez/packages.json
 skare3-test-dashboard -o /proj/sot/ska/jgonzalez/test_results.html --static-dir https://cxc.cfa.harvard.edu/mta/ASPECT/skare3/dashboard/static --log-dir https://icxc.cfa.harvard.edu/aspect/skare3/dashboard/tests
+skare3-test-dashboard -o /proj/sot/ska/jgonzalez/test_results.json
 
 # horrible hack:
 rm -f /proj/sot/ska/www/ASPECT_ICXC/skare3/dashboard/tests
 ln -s `/proj/sot/ska/jgonzalez/git/skare3_tools/scripts/test_dir.py` /proj/sot/ska/www/ASPECT_ICXC/skare3/dashboard/tests
 mv /proj/sot/ska/jgonzalez/index.html /proj/sot/ska/www/ASPECT/skare3/dashboard/
+mv /proj/sot/ska/jgonzalez/packages.json /proj/sot/ska/www/ASPECT/skare3/dashboard/
 mv /proj/sot/ska/jgonzalez/test_results.html /proj/sot/ska/www/ASPECT/skare3/dashboard/tests/index.html
+mv /proj/sot/ska/jgonzalez/test_results.json /proj/sot/ska/www/ASPECT/skare3/dashboard/tests/
 
 # echo files copied to /proj/sot/ska/www/ASPECT/skare3/dashboard
 # echo dashboard at https://cxc.cfa.harvard.edu/mta/ASPECT/skare3/dashboard/

skare3_tools/dashboard/views/dashboard.py

@@ -2,6 +2,8 @@
 
 import argparse
 import datetime
+import json
+from pathlib import Path
 
 from skare3_tools import packages
 from skare3_tools import test_results as tr
@@ -10,7 +12,7 @@
 package_name_map = packages.get_package_list()
 
 
-def dashboard(config=None):
+def dashboard(config=None, render=True):
     if config is None:
         config = {"static_dir": "static"}
 
@@ -60,6 +62,9 @@ def dashboard(config=None):
                 else:
                     p["test_status"] = "PASS"
 
+    if not render:
+        return info
+
     template = get_template("dashboard.html")
     return template.render(title="Skare3 Packages", info=info, config=config)
 
@@ -73,14 +78,18 @@ def get_parser():
         metavar="FILENAME",
         help="Output file (default: index.html)",
         default="index.html",
+        type=Path,
     )
     return parser
 
 
 def main():
     args = get_parser().parse_args()
     with open(args.o, "w") as out:
-        out.write(dashboard())
+        if args.o.suffix == ".json":
+            json.dump(dashboard(render=False), out)
+        else:
+            out.write(dashboard())
 
 
 if __name__ == "__main__":

skare3_tools/dashboard/views/test_results.py

@@ -2,8 +2,8 @@
 
 import argparse
 import json
-import os
 import webbrowser
+from pathlib import Path
 
 from skare3_tools import dashboard
 from skare3_tools import test_results as tr
@@ -15,10 +15,10 @@ def test_results():
         "static_dir": "static",
         "log_dir": "tests/logs/{uid}".format(uid=test_run["run_info"]["uid"]),
     }
-    return _render(test_run, config)
+    return _get_results(test_run, config)
 
 
-def _render(tests, config):
+def _get_results(tests, config, render=True):
     if "run_info" not in tests:
         tests["run_info"] = {
             k: ""
@@ -56,6 +56,9 @@ def _render(tests, config):
         ts["pass"] = n_pass
         ts["fail"] = n_fail
 
+    if not render:
+        return tests
+
     template = dashboard.get_template("test-results.html")
     return template.render(title="Skare3 Tests", data=tests, config=config)
 
@@ -69,13 +72,15 @@ def get_parser():
         help="Directory or JSON file containing all test results. "
         "If it is a directory, then it must have a file named all_tests.json.",
         dest="file_in",
+        type=Path,
     )
     parser.add_argument(
         "-o",
         help="Name of file to write to. By default, this creates a file named "
         "index.html, located in the input directory or the current "
         "working directory, depending on whether the '-i' options was given.",
         dest="file_out",
+        type=Path,
     )
     parser.add_argument(
         "-b",
@@ -96,16 +101,16 @@ def main():
     parser = get_parser()
     args = parser.parse_args()
     config = {"static_dir": args.static_dir, "log_dir": args.log_dir}
-    if args.file_in and os.path.isdir(args.file_in):
-        args.file_in = os.path.join(args.file_in, "all_tests.json")
+    if args.file_in and args.file_in.is_dir():
+        args.file_in = args.file_in / "all_tests.json"
 
     if args.file_out is None:
         if args.file_in:
-            args.file_out = os.path.join(os.path.dirname(args.file_in), "index.html")
+            args.file_out = args.file_in.parent / "index.html"
         else:
             args.file_out = "index.html"
 
-    if args.file_in and not os.path.exists(args.file_in):
+    if args.file_in and not args.file_in.exists():
         print("{filename} does not exist".format(filename=args.file_in))
         parser.print_help()
         parser.exit(1)
@@ -120,10 +125,13 @@ def main():
                 results["run_info"][key] = ", ".join(results["run_info"][key])
 
     with open(args.file_out, "w") as out:
-        out.write(_render(results, config))
+        if args.file_out.suffix == ".json":
+            json.dump(_get_results(results, config, render=False), out)
+        else:
+            out.write(_get_results(results, config))
 
-    if not args.b:
-        file_out = os.path.abspath(args.file_out)
+    if not args.b and args.file_out.suffix in [".html", ".htm"]:
+        file_out = args.file_out.absolute()
         webbrowser.open("file://{file_out}".format(file_out=file_out), new=2)
 
 

skare3_tools/packages.py

@@ -433,7 +433,6 @@ def _get_repository_info_v4(
     for r in releases:
         r["tag_oid"], r["committed_date"] = _get_tag_target(r["tag"])
 
-    releases = {r["tag_oid"]: r for r in releases}
     release_info = [
         {
             "release_tag": "",
@@ -466,17 +465,20 @@ def _get_repository_info_v4(
 
     for commit in commits:
         sha = commit["oid"]
-        if sha in releases:
-            release_info.append(
-                {
-                    "release_sha": sha,
-                    "release_commit_date": releases[sha]["committed_date"],
-                    "release_tag": releases[sha]["tagName"],
-                    "release_tag_date": releases[sha]["publishedAt"],
-                    "commits": [],
-                    "merges": [],
-                }
-            )
+        releases_at_commit = [
+            {
+                "release_sha": release["tag_oid"],
+                "release_commit_date": release["committed_date"],
+                "release_tag": release["tagName"],
+                "release_tag_date": release["publishedAt"],
+                "commits": [],
+                "merges": [],
+            }
+            for release in releases
+            if release["tag_oid"] == sha
+        ]
+        release_info += releases_at_commit
+
         release_info[-1]["commits"].append(commit)
         match = re.match(
             r"Merge pull request #(?P<pr_number>.+) from (?P<branch>\S+)\n\n(?P<title>.+)",
@@ -504,14 +506,14 @@ def _get_repository_info_v4(
     # Now we will add the remaining releases, which presumably happened in another branch.
 
     release_shas = [r["release_sha"] for r in release_info[1:]]
-    for sha in releases:
-        if sha not in release_shas:
+    for release in releases:
+        if release["tag_oid"] not in release_shas:
             release_info.append(
                 {
-                    "release_sha": sha,
-                    "release_tag": releases[sha]["tagName"],
-                    "release_tag_date": releases[sha]["publishedAt"],
-                    "release_commit_date": releases[sha]["committed_date"],
+                    "release_sha": release["tag_oid"],
+                    "release_tag": release["tagName"],
+                    "release_tag_date": release["publishedAt"],
+                    "release_commit_date": release["committed_date"],
                     "commits": [],
                     "merges": [],
                 }
@@ -661,12 +663,15 @@ def _split_versions(depends):
     """
     result = {}
     for depend in depends:
-        v = depend.split("==") if "==" in depend else depend.split()
-        if len(v) > 2:
-            raise Exception(f"Version spec got split into too many parts: {depend}")
-        p_name = v[0].strip()
-        p_version = v[1].strip() if len(v) == 2 else ""
-        result[p_name] = p_version
+        if "==" in depend:
+            name_version = depend.split("==", maxsplit=1)
+        else:
+            name_version = depend.split(maxsplit=1)
+        if len(name_version) == 2:
+            name, version = name_version
+        else:
+            name, version = name_version[0], ""
+        result[name.strip()] = version.strip()
     return result
 
 
@@ -748,9 +753,6 @@ def _get_repository_info_v3(
         for r, c in zip(releases, release_commits)
     }
 
-    # later on, the releases are referred by commit sha
-    releases = {c["sha"]: r for r, c in zip(releases, release_commits)}
-
     date_since = None
     if type(since) is int:
         # only the latest 'since' releases (at most) will be included in summary
@@ -780,15 +782,18 @@ def _get_repository_info_v3(
         commits = commits[:-1]  # remove first commit, which was just the starting point
     for commit in commits:
         sha = commit["sha"]
-        if sha in releases.keys():
-            release_info.append(
-                {
-                    "release_tag": releases[sha]["tag_name"],
-                    "release_tag_date": releases[sha]["published_at"],
-                    "commits": [],
-                    "merges": [],
-                }
-            )
+        releases_at_commit = [
+            {
+                "release_tag": release["tag_name"],
+                "release_tag_date": release["published_at"],
+                "commits": [],
+                "merges": [],
+            }
+            for release in [
+                r for r, c in zip(releases, release_commits) if c["sha"] == sha
+            ]
+        ]
+        release_info += releases_at_commit
 
         release_info[-1]["commits"].append(
             {