diff --git a/.github/workflows/build-metrics.yml b/.github/workflows/build-metrics.yml index 5141dd5d1a64c..366f7a4e5e9dd 100644 --- a/.github/workflows/build-metrics.yml +++ b/.github/workflows/build-metrics.yml @@ -116,3 +116,54 @@ jobs: --pr_number "${{ github.event.number }}" \ --sha "${{ inputs.ref || github.sha }}" \ "/tmp/metrics" + + upload-report: + permissions: + contents: write + runs-on: ubuntu-latest + name: Generate and Upload Build Metric Report + needs: metrics + steps: + - name: Checkout + uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4.1.1 + with: + fetch-depth: 0 + + - name: Setup Git User + run: | + git config --global user.email "velox@users.noreply.github.com" + git config --global user.name "velox" + + - uses: cachix/install-nix-action@8887e596b4ee1134dae06b98d573bd674693f47c # v26 + with: + nix_path: nixpkgs=channel:nixos-unstable + + - name: Build Environment + run: | + cd scripts/bm-report + nix-build + + - name: Build Documentation + env: + CONBENCH_URL: "https://velox-conbench.voltrondata.run/" + CONBENCH_EMAIL: "${{ secrets.CONBENCH_EMAIL }}" + CONBENCH_PASSWORD: "${{ secrets.CONBENCH_PASSWORD }}" + run: | + cd scripts/bm-report + nix-shell --run "quarto render report.qmd" + + - name: Push Report + # The report only uses conbench data from 'main' + # so any data generated in a PR won't be included + if: ${{ github.event_name != 'pull_request' && github.repository == 'facebookincubator/velox'}} + run: | + git checkout gh-pages + mkdir -p docs/bm-report + cp -R scripts/bm-report/report.html docs/bm-report/index.html + git add docs + + if [ -n "$(git status --porcelain --untracked-files=no)" ] + then + git commit -m "Update build metrics" + git push + fi diff --git a/.gitignore b/.gitignore index 243d56b4905a3..614d10dc3f49b 100644 --- a/.gitignore +++ b/.gitignore @@ -321,3 +321,4 @@ src/amalgamation/ #docs velox/docs/sphinx/source/README_generated_* velox/docs/bindings/python/_generate/* +scripts/bm-report/report.html diff --git a/scripts/bm-report/default.nix b/scripts/bm-report/default.nix new file mode 100644 index 0000000000000..4267be1044dd4 --- /dev/null +++ b/scripts/bm-report/default.nix @@ -0,0 +1,61 @@ +# This file was generated by the {rix} R package v0.6.0 on 2024-05-15 +# with following call: +# >rix::rix(r_ver = "abd6d48f8c77bea7dc51beb2adfa6ed3950d2585", +# > r_pkgs = c("dplyr", +# > "prettyunits", +# > "ggplot2", +# > "gh", +# > "gt", +# > "hms", +# > "jqr", +# > "jsonlite", +# > "lubridate", +# > "memoise", +# > "plotly", +# > "purrr", +# > "remotes"), +# > system_pkgs = c("quarto"), +# > git_pkgs = list(package_name = "conbenchcoms", +# > repo_url = "https://github.com/conbench/conbenchcoms", +# > branch_name = "main", +# > commit = "55cdb120bbe2c668d3cf8ae543f4922131653645"), +# > ide = "other", +# > project_path = path_default_nix, +# > overwrite = TRUE, +# > print = TRUE) +# It uses nixpkgs' revision abd6d48f8c77bea7dc51beb2adfa6ed3950d2585 for reproducibility purposes +# which will install R version latest +# Report any issues to https://github.com/b-rodrigues/rix +let + pkgs = import (fetchTarball "https://github.com/NixOS/nixpkgs/archive/abd6d48f8c77bea7dc51beb2adfa6ed3950d2585.tar.gz") {}; + rpkgs = builtins.attrValues { + inherit (pkgs.rPackages) dplyr prettyunits ggplot2 gh gt hms jqr jsonlite lubridate memoise plotly quarto purrr remotes; +}; + git_archive_pkgs = [(pkgs.rPackages.buildRPackage { + name = "conbenchcoms"; + src = pkgs.fetchgit { + url = "https://github.com/conbench/conbenchcoms"; + branchName = "main"; + rev = "55cdb120bbe2c668d3cf8ae543f4922131653645"; + sha256 = "sha256-XR5+grCUKyvSxsqiOkKd3gMUsWDJblZDmF4O+Jehq6U="; + }; + propagatedBuildInputs = builtins.attrValues { + inherit (pkgs.rPackages) dplyr glue httr2 yaml; + }; + }) ]; + system_packages = builtins.attrValues { + inherit (pkgs) R glibcLocales nix quartoMinimal; +}; + in + pkgs.mkShell { + LOCALE_ARCHIVE = if pkgs.system == "x86_64-linux" then "${pkgs.glibcLocales}/lib/locale/locale-archive" else ""; + LANG = "en_US.UTF-8"; + LC_ALL = "en_US.UTF-8"; + LC_TIME = "en_US.UTF-8"; + LC_MONETARY = "en_US.UTF-8"; + LC_PAPER = "en_US.UTF-8"; + LC_MEASUREMENT = "en_US.UTF-8"; + + buildInputs = [ git_archive_pkgs rpkgs system_packages ]; + + } diff --git a/scripts/bm-report/report.qmd b/scripts/bm-report/report.qmd new file mode 100644 index 0000000000000..9951343deec0c --- /dev/null +++ b/scripts/bm-report/report.qmd @@ -0,0 +1,333 @@ +--- +title: "Velox Build Metrics" +execute: + echo: false + warning: false +format: + html: + grid: + sidebar-width: 0px + body-width: 1800px + margin-width: 150px + gutter-width: 1.5rem + self-contained: true + page-layout: full + toc: false + margin-left: 30px + link-external-newwindow: true + theme: cosmo +--- + + + + +```{r setup} +library(gt) +library(ggplot2) +library(plotly) +library(dplyr) +library(purrr) + +# Cache conbench and gh api results for local development +cd <- cachem::cache_disk(rappdirs::user_cache_dir("velox-bm-report")) +mgh <- memoise::memoise(gh::gh, cache = cd) +mruns <- memoise::memoise(conbenchcoms::runs, cache = cd) +mresults <- memoise::memoise(conbenchcoms::benchmark_results, cache = cd) + +# Get latest runs of build-metric job +runs <- mgh( + "GET /repos/facebookincubator/velox/actions/workflows/build-metrics.yml/runs", + status = "success", + branch = "main" +) |> jsonlite::toJSON() + +# Extract the commit sha of the most recent run. The results of the latest +# run are displayed in the tables. +newest_sha <- runs |> + jqr::jq(".workflow_runs | max_by(.updated_at) | .head_sha") |> + jsonlite::fromJSON() + +run_shas <- runs |> + jqr::jq("[.workflow_runs[].head_sha]") |> + jsonlite::fromJSON() +run_ids <- mruns(run_shas) |> + filter(commit.branch == "facebookincubator:main", substr(id, 1, 2) == "BM") |> + pull(id) + +# Fetch the result and do clean/format the data +results <- run_ids |> + purrr::map_df(mresults) |> + mutate( + timestamp = lubridate::as_datetime(timestamp), + stats.data = unlist(stats.data), + type = case_when( + startsWith(run_id, "BM-debug") ~ "debug", + .default = "release" + ) + ) +``` + +```{r ggplot2-specs} +theme_set(theme_minimal(base_size = 12) %+replace% + theme( + plot.title.position = "plot", + strip.text = element_text(size = 12) + )) + +format_tags <- function(x) { + x |> + stringr::str_replace_all("_", " ") |> + stringr::str_to_title() +} +``` + +::::: {.panel-tabset} + +## Times +```{r total-graphs} +# Filter the data and layout the overview plots +times_plot <- results |> + filter(tags.suite == "total", endsWith(tags.source, "time")) |> + mutate( + stats.data = lubridate::dseconds(stats.data), + tags.name = format_tags(tags.name) + ) |> + ggplot(aes( + x = timestamp, + y = stats.data, + group = interaction(tags.name, type), color = tags.name + )) + + facet_wrap(~type) + + geom_line() + + geom_point() + + scale_y_time() + + scale_x_datetime() + + labs( + title = "Velox Build Times", + x = "Date", + y = "Time in Minutes" + ) + + scale_color_viridis_d() +ggplotly(times_plot) |> + layout(legend = list(title = list(text = "Tags Name
"))) ## needed because theme legend specs don't work with ggplotly +``` + +```{r expensive-objects-compile} +# Format compile time data +compile_times <- results |> + filter(tags.suite == "compiling", commit.sha == newest_sha) |> + mutate( + stats.data = lubridate::dseconds(stats.data), + tags.name = glue::glue("`{tags.name}`") + ) +``` + +### Compile Times + +:::: {.columns} + +::: {.column width="49%"} + +```{r compile-times-release} +# Select and format the data to be displayed in the release compile time table +compile_times |> + filter(type == "release") |> + select(tags.name, stats.data) |> + arrange(desc(stats.data)) |> + gt() |> + cols_label( + `tags.name` = "Object", + `stats.data` = "Time" + ) |> + cols_align(align = "left", columns = everything()) |> + tab_header(title = "Release") |> + fmt_markdown(columns = "tags.name") |> + opt_interactive(use_page_size_select = TRUE, use_search = TRUE) +``` + +::: + +::: {.column width="2%"} + +::: + +::: {.column width="49%"} + +```{r compile-times-debug} +# Select and format the data to be displayed in the debug compile time table +compile_times |> + filter(type == "debug") |> + select(tags.name, stats.data) |> + arrange(desc(stats.data)) |> + gt() |> + cols_label( + `tags.name` = "Object", + `stats.data` = "Time" + ) |> + cols_align(align = "left", columns = everything()) |> + tab_header(title = "Debug") |> + fmt_markdown(columns = "tags.name") |> + opt_interactive(use_page_size_select = TRUE, use_search = TRUE) +``` + +::: + +:::: + +```{r expensive-objects-link} +# Format linke time data +link_times <- results |> + filter(tags.suite == "linking", commit.sha == newest_sha) |> + mutate( + stats.data = lubridate::dseconds(stats.data), + tags.name = glue::glue("`{tags.name}`") + ) + +``` + +### Link Times + +:::: {.columns} + +::: {.column width="49%"} + +```{r link-times-release} +# Select and format the data to be displayed in the release link time table +link_times |> + filter(type == "release") |> + select(tags.name, stats.data) |> + arrange(desc(stats.data)) |> + gt() |> + cols_label( + `tags.name` = "Object", + `stats.data` = "Time" + ) |> + cols_align(align = "left", columns = everything()) |> + tab_header(title = "Release") |> + fmt_markdown(columns = "tags.name") |> + opt_interactive(use_page_size_select = TRUE, use_search = TRUE) +``` + +::: + +::: {.column width="2%"} + +::: + +::: {.column width="49%"} + +```{r link-times-debug} +# Select and format the data to be displayed in the debug link time table +link_times |> + filter(type == "debug") |> + select(tags.name, stats.data) |> + arrange(desc(stats.data)) |> + gt() |> + cols_label( + `tags.name` = "Object", + `stats.data` = "Time" + ) |> + cols_align(align = "left", columns = everything()) |> + tab_header(title = "Link Times - Debug") |> + fmt_markdown(columns = "tags.name") |> + opt_interactive(use_page_size_select = TRUE, use_search = TRUE) +``` + +::: + +:::: + + +## Sizes +```{r big-objects} +# This is converts byte values into human-readable values in the tables +size_formatter <- function(x) { + function(x) { + prettyunits::pretty_bytes(x) + } +} + +# Prepare object size data +object_sizes <- results |> + filter(endsWith(tags.source, "size"), commit.sha == newest_sha) |> + mutate( + tags.name = glue::glue("`{tags.name}`") + ) + +# Filter the data and layout the size overview plots +sizes_plot <- results |> + filter(tags.suite == "executable", startsWith(tags.name, "total_")) |> + ggplot(aes( + x = timestamp, + y = stats.data, + group = interaction(tags.name, type), color = tags.name + )) + + facet_wrap(~type) + + geom_line() + + geom_point() + + scale_y_continuous(labels = size_formatter()) + + scale_x_datetime() + + labs( + title = "Velox Object Sizes", + x = "Date", + y = "Size" + ) + + scale_color_viridis_d() +ggplotly(sizes_plot) |> + layout(legend = list(title = list(text = "Tags Name
"))) ## needed because theme legend specs don't work with ggplotly +``` + +### Object Sizes +:::: {.columns} + +::: {.column width="49%"} + +```{r object-sizes-release} +# Select and format the data to be displayed in the release size table +object_sizes |> + filter(type == "release") |> + select(tags.name, stats.data) |> + arrange(desc(stats.data)) |> + gt() |> + cols_label( + `tags.name` = "Object", + `stats.data` = "Size" + ) |> + fmt(columns = `stats.data`, fn = size_formatter()) |> + fmt_markdown(columns = "tags.name") |> + cols_align(align = "left", columns = everything()) |> + tab_header(title = "Release") |> + opt_interactive(use_page_size_select = TRUE, use_search = TRUE) +``` + +::: + +::: {.column width="2%"} + +::: + +::: {.column width="49%"} + +```{r object-sizes-debug} +# Select and format the data to be displayed in the debug size table +object_sizes |> + filter(type == "debug") |> + select(tags.name, stats.data) |> + arrange(desc(stats.data)) |> + gt() |> + fmt(columns = `stats.data`, fn = size_formatter()) |> + fmt_markdown(columns = "tags.name") |> + cols_label( + `tags.name` = "Object", + `stats.data` = "Time" + ) |> + cols_align(align = "left", columns = everything()) |> + tab_header(title = "Debug") |> + opt_interactive(use_page_size_select = TRUE, use_search = TRUE) +``` + +::: + +:::: + +:::::