diff --git a/.github/workflows/build-metrics.yml b/.github/workflows/build-metrics.yml
index 5141dd5d1a64c..366f7a4e5e9dd 100644
--- a/.github/workflows/build-metrics.yml
+++ b/.github/workflows/build-metrics.yml
@@ -116,3 +116,54 @@ jobs:
--pr_number "${{ github.event.number }}" \
--sha "${{ inputs.ref || github.sha }}" \
"/tmp/metrics"
+
+ upload-report:
+ permissions:
+ contents: write
+ runs-on: ubuntu-latest
+ name: Generate and Upload Build Metric Report
+ needs: metrics
+ steps:
+ - name: Checkout
+ uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4.1.1
+ with:
+ fetch-depth: 0
+
+ - name: Setup Git User
+ run: |
+ git config --global user.email "velox@users.noreply.github.com"
+ git config --global user.name "velox"
+
+ - uses: cachix/install-nix-action@8887e596b4ee1134dae06b98d573bd674693f47c # v26
+ with:
+ nix_path: nixpkgs=channel:nixos-unstable
+
+ - name: Build Environment
+ run: |
+ cd scripts/bm-report
+ nix-build
+
+ - name: Build Documentation
+ env:
+ CONBENCH_URL: "https://velox-conbench.voltrondata.run/"
+ CONBENCH_EMAIL: "${{ secrets.CONBENCH_EMAIL }}"
+ CONBENCH_PASSWORD: "${{ secrets.CONBENCH_PASSWORD }}"
+ run: |
+ cd scripts/bm-report
+ nix-shell --run "quarto render report.qmd"
+
+ - name: Push Report
+ # The report only uses conbench data from 'main'
+ # so any data generated in a PR won't be included
+ if: ${{ github.event_name != 'pull_request' && github.repository == 'facebookincubator/velox'}}
+ run: |
+ git checkout gh-pages
+ mkdir -p docs/bm-report
+ cp -R scripts/bm-report/report.html docs/bm-report/index.html
+ git add docs
+
+ if [ -n "$(git status --porcelain --untracked-files=no)" ]
+ then
+ git commit -m "Update build metrics"
+ git push
+ fi
diff --git a/.gitignore b/.gitignore
index 243d56b4905a3..614d10dc3f49b 100644
--- a/.gitignore
+++ b/.gitignore
@@ -321,3 +321,4 @@ src/amalgamation/
#docs
velox/docs/sphinx/source/README_generated_*
velox/docs/bindings/python/_generate/*
+scripts/bm-report/report.html
diff --git a/scripts/bm-report/default.nix b/scripts/bm-report/default.nix
new file mode 100644
index 0000000000000..4267be1044dd4
--- /dev/null
+++ b/scripts/bm-report/default.nix
@@ -0,0 +1,61 @@
+# This file was generated by the {rix} R package v0.6.0 on 2024-05-15
+# with following call:
+# >rix::rix(r_ver = "abd6d48f8c77bea7dc51beb2adfa6ed3950d2585",
+# > r_pkgs = c("dplyr",
+# > "prettyunits",
+# > "ggplot2",
+# > "gh",
+# > "gt",
+# > "hms",
+# > "jqr",
+# > "jsonlite",
+# > "lubridate",
+# > "memoise",
+# > "plotly",
+# > "purrr",
+# > "remotes"),
+# > system_pkgs = c("quarto"),
+# > git_pkgs = list(package_name = "conbenchcoms",
+# > repo_url = "https://github.com/conbench/conbenchcoms",
+# > branch_name = "main",
+# > commit = "55cdb120bbe2c668d3cf8ae543f4922131653645"),
+# > ide = "other",
+# > project_path = path_default_nix,
+# > overwrite = TRUE,
+# > print = TRUE)
+# It uses nixpkgs' revision abd6d48f8c77bea7dc51beb2adfa6ed3950d2585 for reproducibility purposes
+# which will install R version latest
+# Report any issues to https://github.com/b-rodrigues/rix
+let
+ pkgs = import (fetchTarball "https://github.com/NixOS/nixpkgs/archive/abd6d48f8c77bea7dc51beb2adfa6ed3950d2585.tar.gz") {};
+ rpkgs = builtins.attrValues {
+ inherit (pkgs.rPackages) dplyr prettyunits ggplot2 gh gt hms jqr jsonlite lubridate memoise plotly quarto purrr remotes;
+};
+ git_archive_pkgs = [(pkgs.rPackages.buildRPackage {
+ name = "conbenchcoms";
+ src = pkgs.fetchgit {
+ url = "https://github.com/conbench/conbenchcoms";
+ branchName = "main";
+ rev = "55cdb120bbe2c668d3cf8ae543f4922131653645";
+ sha256 = "sha256-XR5+grCUKyvSxsqiOkKd3gMUsWDJblZDmF4O+Jehq6U=";
+ };
+ propagatedBuildInputs = builtins.attrValues {
+ inherit (pkgs.rPackages) dplyr glue httr2 yaml;
+ };
+ }) ];
+ system_packages = builtins.attrValues {
+ inherit (pkgs) R glibcLocales nix quartoMinimal;
+};
+ in
+ pkgs.mkShell {
+ LOCALE_ARCHIVE = if pkgs.system == "x86_64-linux" then "${pkgs.glibcLocales}/lib/locale/locale-archive" else "";
+ LANG = "en_US.UTF-8";
+ LC_ALL = "en_US.UTF-8";
+ LC_TIME = "en_US.UTF-8";
+ LC_MONETARY = "en_US.UTF-8";
+ LC_PAPER = "en_US.UTF-8";
+ LC_MEASUREMENT = "en_US.UTF-8";
+
+ buildInputs = [ git_archive_pkgs rpkgs system_packages ];
+
+ }
diff --git a/scripts/bm-report/report.qmd b/scripts/bm-report/report.qmd
new file mode 100644
index 0000000000000..9951343deec0c
--- /dev/null
+++ b/scripts/bm-report/report.qmd
@@ -0,0 +1,333 @@
+---
+title: "Velox Build Metrics"
+execute:
+ echo: false
+ warning: false
+format:
+ html:
+ grid:
+ sidebar-width: 0px
+ body-width: 1800px
+ margin-width: 150px
+ gutter-width: 1.5rem
+ self-contained: true
+ page-layout: full
+ toc: false
+ margin-left: 30px
+ link-external-newwindow: true
+ theme: cosmo
+---
+
+
+
+
+```{r setup}
+library(gt)
+library(ggplot2)
+library(plotly)
+library(dplyr)
+library(purrr)
+
+# Cache conbench and gh api results for local development
+cd <- cachem::cache_disk(rappdirs::user_cache_dir("velox-bm-report"))
+mgh <- memoise::memoise(gh::gh, cache = cd)
+mruns <- memoise::memoise(conbenchcoms::runs, cache = cd)
+mresults <- memoise::memoise(conbenchcoms::benchmark_results, cache = cd)
+
+# Get latest runs of build-metric job
+runs <- mgh(
+ "GET /repos/facebookincubator/velox/actions/workflows/build-metrics.yml/runs",
+ status = "success",
+ branch = "main"
+) |> jsonlite::toJSON()
+
+# Extract the commit sha of the most recent run. The results of the latest
+# run are displayed in the tables.
+newest_sha <- runs |>
+ jqr::jq(".workflow_runs | max_by(.updated_at) | .head_sha") |>
+ jsonlite::fromJSON()
+
+run_shas <- runs |>
+ jqr::jq("[.workflow_runs[].head_sha]") |>
+ jsonlite::fromJSON()
+run_ids <- mruns(run_shas) |>
+ filter(commit.branch == "facebookincubator:main", substr(id, 1, 2) == "BM") |>
+ pull(id)
+
+# Fetch the result and do clean/format the data
+results <- run_ids |>
+ purrr::map_df(mresults) |>
+ mutate(
+ timestamp = lubridate::as_datetime(timestamp),
+ stats.data = unlist(stats.data),
+ type = case_when(
+ startsWith(run_id, "BM-debug") ~ "debug",
+ .default = "release"
+ )
+ )
+```
+
+```{r ggplot2-specs}
+theme_set(theme_minimal(base_size = 12) %+replace%
+ theme(
+ plot.title.position = "plot",
+ strip.text = element_text(size = 12)
+ ))
+
+format_tags <- function(x) {
+ x |>
+ stringr::str_replace_all("_", " ") |>
+ stringr::str_to_title()
+}
+```
+
+::::: {.panel-tabset}
+
+## Times
+```{r total-graphs}
+# Filter the data and layout the overview plots
+times_plot <- results |>
+ filter(tags.suite == "total", endsWith(tags.source, "time")) |>
+ mutate(
+ stats.data = lubridate::dseconds(stats.data),
+ tags.name = format_tags(tags.name)
+ ) |>
+ ggplot(aes(
+ x = timestamp,
+ y = stats.data,
+ group = interaction(tags.name, type), color = tags.name
+ )) +
+ facet_wrap(~type) +
+ geom_line() +
+ geom_point() +
+ scale_y_time() +
+ scale_x_datetime() +
+ labs(
+ title = "Velox Build Times",
+ x = "Date",
+ y = "Time in Minutes"
+ ) +
+ scale_color_viridis_d()
+ggplotly(times_plot) |>
+ layout(legend = list(title = list(text = "Tags Name
"))) ## needed because theme legend specs don't work with ggplotly
+```
+
+```{r expensive-objects-compile}
+# Format compile time data
+compile_times <- results |>
+ filter(tags.suite == "compiling", commit.sha == newest_sha) |>
+ mutate(
+ stats.data = lubridate::dseconds(stats.data),
+ tags.name = glue::glue("`{tags.name}`")
+ )
+```
+
+### Compile Times
+
+:::: {.columns}
+
+::: {.column width="49%"}
+
+```{r compile-times-release}
+# Select and format the data to be displayed in the release compile time table
+compile_times |>
+ filter(type == "release") |>
+ select(tags.name, stats.data) |>
+ arrange(desc(stats.data)) |>
+ gt() |>
+ cols_label(
+ `tags.name` = "Object",
+ `stats.data` = "Time"
+ ) |>
+ cols_align(align = "left", columns = everything()) |>
+ tab_header(title = "Release") |>
+ fmt_markdown(columns = "tags.name") |>
+ opt_interactive(use_page_size_select = TRUE, use_search = TRUE)
+```
+
+:::
+
+::: {.column width="2%"}
+
+:::
+
+::: {.column width="49%"}
+
+```{r compile-times-debug}
+# Select and format the data to be displayed in the debug compile time table
+compile_times |>
+ filter(type == "debug") |>
+ select(tags.name, stats.data) |>
+ arrange(desc(stats.data)) |>
+ gt() |>
+ cols_label(
+ `tags.name` = "Object",
+ `stats.data` = "Time"
+ ) |>
+ cols_align(align = "left", columns = everything()) |>
+ tab_header(title = "Debug") |>
+ fmt_markdown(columns = "tags.name") |>
+ opt_interactive(use_page_size_select = TRUE, use_search = TRUE)
+```
+
+:::
+
+::::
+
+```{r expensive-objects-link}
+# Format linke time data
+link_times <- results |>
+ filter(tags.suite == "linking", commit.sha == newest_sha) |>
+ mutate(
+ stats.data = lubridate::dseconds(stats.data),
+ tags.name = glue::glue("`{tags.name}`")
+ )
+
+```
+
+### Link Times
+
+:::: {.columns}
+
+::: {.column width="49%"}
+
+```{r link-times-release}
+# Select and format the data to be displayed in the release link time table
+link_times |>
+ filter(type == "release") |>
+ select(tags.name, stats.data) |>
+ arrange(desc(stats.data)) |>
+ gt() |>
+ cols_label(
+ `tags.name` = "Object",
+ `stats.data` = "Time"
+ ) |>
+ cols_align(align = "left", columns = everything()) |>
+ tab_header(title = "Release") |>
+ fmt_markdown(columns = "tags.name") |>
+ opt_interactive(use_page_size_select = TRUE, use_search = TRUE)
+```
+
+:::
+
+::: {.column width="2%"}
+
+:::
+
+::: {.column width="49%"}
+
+```{r link-times-debug}
+# Select and format the data to be displayed in the debug link time table
+link_times |>
+ filter(type == "debug") |>
+ select(tags.name, stats.data) |>
+ arrange(desc(stats.data)) |>
+ gt() |>
+ cols_label(
+ `tags.name` = "Object",
+ `stats.data` = "Time"
+ ) |>
+ cols_align(align = "left", columns = everything()) |>
+ tab_header(title = "Link Times - Debug") |>
+ fmt_markdown(columns = "tags.name") |>
+ opt_interactive(use_page_size_select = TRUE, use_search = TRUE)
+```
+
+:::
+
+::::
+
+
+## Sizes
+```{r big-objects}
+# This is converts byte values into human-readable values in the tables
+size_formatter <- function(x) {
+ function(x) {
+ prettyunits::pretty_bytes(x)
+ }
+}
+
+# Prepare object size data
+object_sizes <- results |>
+ filter(endsWith(tags.source, "size"), commit.sha == newest_sha) |>
+ mutate(
+ tags.name = glue::glue("`{tags.name}`")
+ )
+
+# Filter the data and layout the size overview plots
+sizes_plot <- results |>
+ filter(tags.suite == "executable", startsWith(tags.name, "total_")) |>
+ ggplot(aes(
+ x = timestamp,
+ y = stats.data,
+ group = interaction(tags.name, type), color = tags.name
+ )) +
+ facet_wrap(~type) +
+ geom_line() +
+ geom_point() +
+ scale_y_continuous(labels = size_formatter()) +
+ scale_x_datetime() +
+ labs(
+ title = "Velox Object Sizes",
+ x = "Date",
+ y = "Size"
+ ) +
+ scale_color_viridis_d()
+ggplotly(sizes_plot) |>
+ layout(legend = list(title = list(text = "Tags Name
"))) ## needed because theme legend specs don't work with ggplotly
+```
+
+### Object Sizes
+:::: {.columns}
+
+::: {.column width="49%"}
+
+```{r object-sizes-release}
+# Select and format the data to be displayed in the release size table
+object_sizes |>
+ filter(type == "release") |>
+ select(tags.name, stats.data) |>
+ arrange(desc(stats.data)) |>
+ gt() |>
+ cols_label(
+ `tags.name` = "Object",
+ `stats.data` = "Size"
+ ) |>
+ fmt(columns = `stats.data`, fn = size_formatter()) |>
+ fmt_markdown(columns = "tags.name") |>
+ cols_align(align = "left", columns = everything()) |>
+ tab_header(title = "Release") |>
+ opt_interactive(use_page_size_select = TRUE, use_search = TRUE)
+```
+
+:::
+
+::: {.column width="2%"}
+
+:::
+
+::: {.column width="49%"}
+
+```{r object-sizes-debug}
+# Select and format the data to be displayed in the debug size table
+object_sizes |>
+ filter(type == "debug") |>
+ select(tags.name, stats.data) |>
+ arrange(desc(stats.data)) |>
+ gt() |>
+ fmt(columns = `stats.data`, fn = size_formatter()) |>
+ fmt_markdown(columns = "tags.name") |>
+ cols_label(
+ `tags.name` = "Object",
+ `stats.data` = "Time"
+ ) |>
+ cols_align(align = "left", columns = everything()) |>
+ tab_header(title = "Debug") |>
+ opt_interactive(use_page_size_select = TRUE, use_search = TRUE)
+```
+
+:::
+
+::::
+
+:::::