Add metrics table download buttons (#135)

* write metric table variants as PDFs and SVGs to site/static/figs

* update site-internal terminology for hidden-by-default models from closed-source to non-compliant

* add PDF and SVG static site assets for all metrics table variants

* readme add how to cite info

* CaptionedMetricsTable.svelte add download as PDF/SVG buttons

data/mp/get_mp_traj.py

@@ -118,7 +118,7 @@
 # %% use gzip CLI to check all files for archive corruption
 for path in tqdm(glob(f"{module_dir}/mp-tasks/*.json.gz")):
     try:
-        subprocess.run(["gzip", "--test", path], check=True)  # noqa: S607
+        subprocess.run(["gzip", "--test", path], check=True)
     except subprocess.CalledProcessError as exc:
         print(f"{path} raised {exc.stderr}")
         # ask user to delete corrupted file

models/orb/orb.yml

@@ -25,7 +25,7 @@ authors: # required (only name, other keys are optional)
   - name: Arthur Hussey
     affiliation: Orbital Materials
 
-repo: https://github.com/orbital-materials/orb-models/
+repo: https://github.com/orbital-materials/orb-models
 url: "#" # placeholder
 doi: "#" # placeholder
 paper: "#" # placeholder

pyproject.toml

@@ -106,7 +106,8 @@ ignore = [
   "S108",
   "S310",
   "S311",
-  "S603",
+  "S603",    # subprocess-without-shell-equals-true
+  "S607",    # start-process-with-partial-path
   "T201",
   "TD",
   "TRY003",

readme.md

@@ -25,7 +25,11 @@ Our results show that ML models have become robust enough to deploy them as tria
 
 <slot name="metrics-table" />
 
-We welcome contributions that add new models to the leaderboard through GitHub PRs. See the [contributing guide](https://janosh.github.io/matbench-discovery/contribute) for details.
+If you'd like to refer to Matbench Discovery in a publication, please cite the [preprint](https://doi.org/10.48550/arXiv.2308.14920):
+
+> Riebesell, Janosh, Rhys E. A. Goodall, Philipp Benner, Yuan Chiang, Bowen Deng, Alpha A. Lee, Anubhav Jain, and Kristin A. Persson. "Matbench Discovery -- A Framework to Evaluate Machine Learning Crystal Stability Predictions." arXiv, August 28, 2023. https://doi.org/10.48550/arXiv.2308.14920.
+
+We welcome new models additions to the leaderboard through GitHub PRs. See the [contributing guide](https://janosh.github.io/matbench-discovery/contribute) for details.
 
 If you're interested in joining this work, please reach out via [GitHub discussion](https://github.com/janosh/matbench-discovery/discussions) or [email](mailto:[email protected]?subject=Collaborate%20on%20Matbench%20Discovery).
 

scripts/model_figs/metrics_tables.py

@@ -1,4 +1,8 @@
 # %%
+import itertools
+import subprocess
+from glob import glob
+
 import numpy as np
 import pandas as pd
 import yaml
@@ -7,7 +11,7 @@
 from pymatviz.utils import si_fmt
 from sklearn.dummy import DummyClassifier
 
-from matbench_discovery import DATA_DIR, PDF_FIGS, SCRIPTS, SITE_FIGS
+from matbench_discovery import DATA_DIR, PDF_FIGS, ROOT, SCRIPTS, SITE_FIGS
 from matbench_discovery.data import DataFiles, df_wbm
 from matbench_discovery.enums import MbdKey, Open
 from matbench_discovery.metrics import stable_metrics
@@ -33,11 +37,11 @@
 date_added_col = "Date Added"
 df_met.loc[Key.train_set.label] = df_met.loc[date_added_col] = ""
 
-hide_closed = False  # hide proprietary models (openness != OSOD)
-closed_models = [
+non_compliant_models = [
     key
     for key, meta in MODEL_METADATA.items()
-    if meta.get("openness", Open.OSOD) != Open.OSOD
+    if meta.get("openness", Open.OSOD)
+    != Open.OSOD  # TODO add `or uses_extra_training_data`
 ]
 
 with open(f"{DATA_DIR}/training-sets.yml") as file:
@@ -51,7 +55,7 @@
 
     # Add model version as hover tooltip to model name
     model_version = model_metadata.get("model_version", "")
-    css_cls = "proprietary" if model in closed_models else ""
+    css_cls = "non-compliant" if model in non_compliant_models else ""
     attrs = {"title": f"Version: {model_version}", "class": css_cls}
     html_attr_str = " ".join(f'{k}="{v}"' for k, v in attrs.items() if v)
     df_met.loc[Key.model_name.label, model] = f"<span {html_attr_str}>{model}</span>"
@@ -205,18 +209,19 @@
 ]
 show_cols = [*f"F1,DAF,Prec,Acc,TPR,TNR,MAE,RMSE,{R2_col}".split(","), *meta_cols]
 
-for label, df_met in (
-    ("", df_metrics),
-    ("-first-10k", df_metrics_10k),
-    ("-uniq-protos", df_metrics_uniq_protos),
+for (label, df_met), show_non_compliant in itertools.product(
+    (
+        ("", df_metrics),
+        ("-first-10k", df_metrics_10k),
+        ("-uniq-protos", df_metrics_uniq_protos),
+    ),
+    (True, False),
 ):
     # abbreviate long column names
     df_met = df_met.rename(index={"R2": R2_col, "Precision": "Prec", "Accuracy": "Acc"})
     df_met.index.name = "Model"
     # only keep columns we want to show
-    df_table = df_met.drop(columns=closed_models if hide_closed else []).T.filter(
-        show_cols
-    )
+    df_table = df_met.T.filter(show_cols)
     df_table = df_table.set_index(Key.model_name.label)
     df_table.index.name = None
 
@@ -276,15 +281,37 @@
         # draw line between classification and regression metrics
         styles=f"{col_selector} {{ border-left: 1px solid white; }}{hide_scroll_bar}",
     )
-    suffix = "" if hide_closed else "-with-closed"
+    suffix = "" if show_non_compliant else "-only-compliant"
+    non_compliant_idx = [  # get index HTML strings of non-compliant models
+        idx
+        for idx in styler.index
+        if any(f">{model_name}<" in idx for model_name in non_compliant_models)
+    ]
     try:
-        df_to_pdf(styler, f"{PDF_FIGS}/metrics-table{label}{suffix}.pdf")
+        for pdf_path in (PDF_FIGS, f"{ROOT}/site/static/figs"):
+            df_to_pdf(
+                styler.hide([] if show_non_compliant else non_compliant_idx),
+                f"{pdf_path}/metrics-table{label}{suffix}.pdf",
+            )
     except (ImportError, RuntimeError) as exc:
         print(f"df_to_pdf failed: {exc}")
 
     display(styler.set_caption(df_met.attrs.get("title")))
 
 
+try:
+    # convert PDFs in site/static/figs to SVGs
+    for pdf_path in glob(f"{ROOT}/site/static/figs/metrics-table*.pdf"):
+        subprocess.run(
+            ["pdf2svg", pdf_path, pdf_path.replace(".pdf", ".svg")], check=False
+        )
+
+    # svgo compress SVGs
+    subprocess.run(["svgo", "--multipass", f"{ROOT}/site/static/figs"], check=False)
+except FileNotFoundError:  # skip in CI where pdf2svg and svgo not installed
+    pass
+
+
 # %% PNG metrics table unused
 if False:
     try:

site/src/lib/CaptionedMetricsTable.svelte

@@ -2,14 +2,23 @@
   import MetricsTable from '$figs/metrics-table-uniq-protos.svelte'
   import { pretty_num } from 'elementari'
 
-  export let show_proprietary = false
+  export let show_non_compliant = false
 
   let n_wbm_stable_uniq_protos = 32_942
   let n_wbm_uniq_protos = 215_488
 </script>
 
-<figure {...$$props} class:hide-prop={!show_proprietary}>
+<figure {...$$props} class:nide-non-compliant={!show_non_compliant}>
   <MetricsTable />
+  <div class="downloads">
+    Download table as
+    {#each ['PDF', 'SVG'] as file_ext}
+      {@const suffix = show_non_compliant ? '' : '-only-compliant'}
+      <a href="/figs/metrics-table-uniq-protos{suffix}.{file_ext.toLowerCase()}" download>
+        {file_ext}</a
+      >
+    {/each}
+  </div>
   <figcaption>
     Training size is the number of materials used to train the model. For models trained
     on DFT relaxations, we show the number of distinct frames in parentheses. In cases
@@ -39,8 +48,19 @@
     padding: 2pt 6pt;
     background-color: rgba(255, 255, 255, 0.07);
   }
-  /* hide rows (<tr>) where any cell has a class of proprietary */
-  figure.hide-prop :global(tr:has(.proprietary)) {
+  /* hide rows (<tr>) where any cell has a class of non-compliant */
+  figure.nide-non-compliant :global(tr:has(.non-compliant)) {
     display: none;
   }
+  div.downloads {
+    display: flex;
+    gap: 1ex;
+    justify-content: center;
+    margin: 1ex 0;
+  }
+  div.downloads a {
+    background-color: rgba(255, 255, 255, 0.1);
+    padding: 0 6pt;
+    border-radius: 4pt;
+  }
 </style>

site/src/routes/+page.svelte

@@ -7,7 +7,7 @@
   import Icon from '@iconify/svelte'
   import { Toggle, Tooltip } from 'svelte-zoo'
 
-  let show_proprietary = false
+  let show_non_compliant = false
 
   const metadata = import.meta.glob(`$root/models/**/*.yml`, {
     eager: true,
@@ -23,7 +23,7 @@
     }
 
     const openness = md.openness ?? `OSOD`
-    if ((!best?.F1 || stats?.F1 > best?.F1) && (show_proprietary || openness == `OSOD`))
+    if ((!best?.F1 || stats?.F1 > best?.F1) && (show_non_compliant || openness == `OSOD`))
       return { ...stats, ...md } as ModelData
 
     return best
@@ -36,16 +36,16 @@
   <div slot="best-report">
     {#if best_model}
       {@const { model_name, F1, R2, DAF, repo, paper } = best_model}
-      We find <a href={repo}>{model_name}</a> (<a href={paper}>paper</a>) to achieve the
-      highest F1 score of {F1}, R<sup>2</sup> of {R2}
+      <a href={repo}>{model_name}</a> (<a href={paper}>paper</a>) achieves the highest F1
+      score of {F1}, R<sup>2</sup> of {R2}
       and a discovery acceleration factor (DAF) of {DAF}
       (i.e. a ~{Number(DAF).toFixed(1)}x higher rate of stable structures compared to
       dummy discovery in the already enriched test set containing 16% stable materials).
     {/if}
   </div>
 
   <div slot="metrics-table" style="display: grid; gap: 1ex; place-items: center;">
-    <Toggle bind:checked={show_proprietary}
+    <Toggle bind:checked={show_non_compliant}
       >Show non-compliant models <Tooltip max_width="10em">
         <span slot="tip">
           Models can be non-compliant for multiple reasons<br />
@@ -61,6 +61,6 @@
         <Icon icon="octicon:info-16" inline style="padding: 0 3pt;" />
       </Tooltip>&ensp;</Toggle
     >
-    <CaptionedMetricsTable bind:show_proprietary />
+    <CaptionedMetricsTable bind:show_non_compliant />
   </div>
 </Readme>

site/src/routes/models/+page.svelte

@@ -14,14 +14,14 @@
   export let data
 
   let sort_by: keyof ModelStats | 'model_name' = `F1`
-  let show_proprietary: boolean = false
+  let show_non_compliant: boolean = false
   let show_details: boolean = false
   let order: 'asc' | 'desc' = `desc`
   let show_n_best: number = data.models.length // show only best models
   const min_models: number = 2
 
   $: models = data.models
-    .filter((model) => show_proprietary || (model.openness ?? `OSOD`) == `OSOD`)
+    .filter((model) => show_non_compliant || (model.openness ?? `OSOD`) == `OSOD`)
     .sort((model_1, model_2) => {
       const [val_1, val_2] = [model_1[sort_by], model_2[sort_by]]
       if (typeof val_1 == `string`) {
@@ -75,7 +75,7 @@
   </p>
 
   <span>
-    <Toggle bind:checked={show_proprietary}>Show proprietary models&ensp;</Toggle>
+    <Toggle bind:checked={show_non_compliant}>Show non-compliant models&ensp;</Toggle>
     &emsp;&emsp; Sort
     <input type="number" min={min_models} max={models.length} bind:value={show_n_best} />
     best models