From 8fbec0079bf9d8787fd1941568a45903bcf9c331 Mon Sep 17 00:00:00 2001 From: Jacob Wujciak-Jens Date: Wed, 1 May 2024 05:27:06 +0200 Subject: [PATCH 1/9] barebones overview --- scripts/bm-report/report.Rmd | 221 + scripts/bm-report/report.html | 8658 +++++++++++++++++++++++++++++++++ 2 files changed, 8879 insertions(+) create mode 100644 scripts/bm-report/report.Rmd create mode 100644 scripts/bm-report/report.html diff --git a/scripts/bm-report/report.Rmd b/scripts/bm-report/report.Rmd new file mode 100644 index 000000000000..9b1ca47544a2 --- /dev/null +++ b/scripts/bm-report/report.Rmd @@ -0,0 +1,221 @@ +--- +title: "Velox Build Metrics" +execute: + warning: false + echo: false +format: + html: + grid: + sidebar-width: 0px + body-width: 1800px + margin-width: 150px + gutter-width: 1.5rem + self-contained: true + page-layout: full + toc: false + margin-left: 30px + link-external-newwindow: true + theme: cosmo +--- + +```{r setup} +library(gt) +library(ggplot2) +library(plotly) +library(gdata) +library(dplyr) + +cd <- cachem::cache_disk(rappdirs::user_cache_dir("velox-bm-report")) +mgh <- memoise::memoise(gh::gh, cache = cd) +mruns <- memoise::memoise(conbenchcoms::runs, cache = cd) +mresults <- memoise::memoise(conbenchcoms::benchmark_results, cache = cd) + +runs <- mgh( + "GET /repos/facebookincubator/velox/actions/workflows/build-metrics.yml/runs", + status = "success", + branch = "main" +) |> jsonlite::toJSON() + +newest_sha <- runs |> + jqr::jq(".workflow_runs | max_by(.updated_at) | .head_sha") |> + jsonlite::fromJSON() + +run_shas <- runs |> + jqr::jq("[.workflow_runs[].head_sha]") |> + jsonlite::fromJSON() +run_ids <- mruns(run_shas) |> + filter(commit.branch == "facebookincubator:main", substr(id, 1, 2) == "BM") |> + pull(id) +results <- run_ids |> + purrr::map_df(mresults) |> + mutate( + timestamp = lubridate::as_datetime(timestamp), + stats.data = unlist(stats.data), + type = case_when( + startsWith(run_id, "BM-debug") ~ "debug", + .default = "release" + ) + ) +``` +::: {.panel-tabset} +## Times +```{r total-graphs} +times_plot <- results |> + filter(tags.suite == "total", endsWith(tags.source, "time")) |> + mutate( + stats.data = lubridate::dseconds(stats.data), + ) |> + ggplot(aes( + x = timestamp, + y = stats.data, + group = interaction(tags.name, type), color = tags.name + )) + + facet_wrap(~type) + + geom_line() + + geom_point() + + scale_y_time() + + scale_x_datetime() + + labs( + title = "Velox Build Times", + x = "Date", + y = "Time in Minutes" + ) + + theme_minimal() +ggplotly(times_plot) +``` +```{r expensive-objects} +compile_times <- results |> + filter(tags.suite == "compiling", commit.sha == newest_sha) |> + mutate( + stats.data = lubridate::dseconds(stats.data), + ) +compile_times |> + filter(type == "release") |> + select(tags.name, stats.data) |> + arrange(desc(stats.data)) |> + gt() |> + cols_label( + `tags.name` = "Object", + `stats.data` = "Time" + ) |> + cols_align(align = "left", columns = everything()) |> + tab_header(title = "Compile Times - Release") |> + opt_interactive(use_page_size_select = TRUE, use_search = TRUE) + +compile_times |> + filter(type == "debug") |> + select(tags.name, stats.data) |> + arrange(desc(stats.data)) |> + gt() |> + cols_label( + `tags.name` = "Object", + `stats.data` = "Time" + ) |> + cols_align(align = "left", columns = everything()) |> + tab_header(title = "Compile Times - Debug") |> + opt_interactive(use_page_size_select = TRUE, use_search = TRUE) + +link_times <- results |> + filter(tags.suite == "linking", commit.sha == newest_sha) |> + mutate( + stats.data = lubridate::dseconds(stats.data), + ) + +link_times |> + filter(type == "release") |> + select(tags.name, stats.data) |> + arrange(desc(stats.data)) |> + gt() |> + cols_label( + `tags.name` = "Object", + `stats.data` = "Time" + ) |> + cols_align(align = "left", columns = everything()) |> + tab_header(title = "Link Times - Release") |> + opt_interactive(use_page_size_select = TRUE, use_search = TRUE) + +link_times |> + filter(type == "debug") |> + select(tags.name, stats.data) |> + arrange(desc(stats.data)) |> + gt() |> + cols_label( + `tags.name` = "Object", + `stats.data` = "Time" + ) |> + cols_align(align = "left", columns = everything()) |> + tab_header(title = "Link Times - Debug") |> + opt_interactive(use_page_size_select = TRUE, use_search = TRUE) +``` +## Sizes +```{r big-objects} +size_formatter <- function() { + function(x) { + gdata::humanReadable(x, standard = "Unix") + } +} +create_byte <- function(bytes) { + stopifnot(is.numeric(bytes)) + structure( + list(bytes = bytes), + class = "BytesObject" + ) +} +Bytes <- Vectorize(create_byte, SIMPLIFY = FALSE) + +print.BytesObject <- function(x, ...) { + human_readable <- prettyunits::pretty_bytes(x$bytes) + cat(human_readable) +} + +object_sizes <- results |> + filter(endsWith(tags.source, "size"), commit.sha == newest_sha) +sizes_plot <- results |> + filter(tags.suite == "executable", startsWith(tags.name, "total_")) |> + ggplot(aes( + x = timestamp, + y = stats.data, + group = interaction(tags.name, type), color = tags.name + )) + + facet_wrap(~type) + + geom_line() + + geom_point() + + scale_y_continuous(labels = size_formatter()) + + scale_x_datetime() + + labs( + title = "Velox Object Sizes", + x = "Date", + y = "Size" + ) + + theme_minimal() +ggplotly(sizes_plot) + +object_sizes |> + filter(type == "release") |> + select(tags.name, stats.data) |> + arrange(desc(stats.data)) |> + gt() |> + cols_label( + `tags.name` = "Object", + `stats.data` = "Size" + ) |> + fmt(columns = `stats.data`, fn = size_formatter()) |> + cols_align(align = "left", columns = everything()) |> + tab_header(title = "Object Sizes - Release") |> + opt_interactive(use_page_size_select = TRUE, use_search = TRUE) + +object_sizes |> + filter(type == "debug") |> + select(tags.name, stats.data) |> + arrange(desc(stats.data)) |> + gt() |> + fmt(columns = `stats.data`, fn = size_formatter()) |> + cols_label( + `tags.name` = "Object", + `stats.data` = "Time" + ) |> + cols_align(align = "left", columns = everything()) |> + tab_header(title = "Object Sizes - Debug") |> + opt_interactive(use_page_size_select = TRUE, use_search = TRUE) +``` +::: diff --git a/scripts/bm-report/report.html b/scripts/bm-report/report.html new file mode 100644 index 000000000000..a6470f7a2ca6 --- /dev/null +++ b/scripts/bm-report/report.html @@ -0,0 +1,8658 @@ + + + + + + + + + +Velox Build Metrics + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
+ +
+ +
+
+

Velox Build Metrics

+
+ + + +
+ + + + +
+ + + +
+ + +
+ +
+
+
+
+
+ +
+
+
+
+
+ +
+
Compile Times - Release
+
+
+
+ +
+
+
+
+ +
+
Compile Times - Debug
+
+
+
+ +
+
+
+
+ +
+
Link Times - Release
+
+
+
+ +
+
+
+
+ +
+
Link Times - Debug
+
+
+
+ +
+
+
+
+
+
+
+
+ +
+
+
+ +
+
Object Sizes - Release
+
+
+
+ +
+
+
+
+ +
+
Object Sizes - Debug
+
+
+
+ +
+
+
+
+
+
+ +
+ + +
+ + + + + \ No newline at end of file From 794e318c97f08b6331a1cf46f7bc4a5b54525baa Mon Sep 17 00:00:00 2001 From: Jacob Wujciak-Jens Date: Tue, 14 May 2024 04:09:27 +0200 Subject: [PATCH 2/9] ignore rendered report clean up add job to publish build metrics add missing package fix typo add missing package add gh token for remotes add credentials add missing suggested package for scales --- .github/workflows/build-metrics.yml | 70 + .gitignore | 1 + scripts/bm-report/report.Rmd | 13 - scripts/bm-report/report.html | 8658 --------------------------- 4 files changed, 71 insertions(+), 8671 deletions(-) delete mode 100644 scripts/bm-report/report.html diff --git a/.github/workflows/build-metrics.yml b/.github/workflows/build-metrics.yml index 5141dd5d1a64..1a581ea41bc1 100644 --- a/.github/workflows/build-metrics.yml +++ b/.github/workflows/build-metrics.yml @@ -116,3 +116,73 @@ jobs: --pr_number "${{ github.event.number }}" \ --sha "${{ inputs.ref || github.sha }}" \ "/tmp/metrics" + + upload-report: + permissions: + contents: write + runs-on: ubuntu-latest + name: Generate and Upload Build Metric Report + # needs: metrics + steps: + - name: Checkout + uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4.1.1 + with: + fetch-depth: 0 + + - name: Setup Git User + run: | + git config --global user.email "velox@users.noreply.github.com" + git config --global user.name "velox" + + - uses: quarto-dev/quarto-actions/setup@c70b2f816b8d51a4badea651aee0602de0ad87bf + - uses: r-lib/actions/setup-r@fbafc3bc4ba114e72680c71e835c59b022606c46 + with: + install-r: false + use-public-rspm: true + + - name: Install Dependencies + env: + GITHUB_PAT: ${{ github.token }} + shell: Rscript {0} + run: | + install.packages( + c( + "dplyr", + "gdata", + "ggplot2", + "gh", + "gt", + "hms", + "jqr", + "jsonlite", + "lubridate", + "memoise", + "plotly", + "purrr", + "remotes" + )) + remotes::install_github("conbench/conbenchcoms") + + - name: Build Documentation + env: + CONBENCH_URL: "https://velox-conbench.voltrondata.run/" + CONBENCH_EMAIL: "${{ secrets.CONBENCH_EMAIL }}" + CONBENCH_PASSWORD: "${{ secrets.CONBENCH_PASSWORD }}" + + run: | + cd scripts/bm-report + quarto render report.Rmd + + - name: Push Documentation + # if: ${{ github.event_name == 'push' && github.repository == 'facebookincubator/velox'}} + run: | + git checkout gh-pages + mkdir -p docs/bm-report + cp -R scripts/bm-report/report.html docs/bm-report/index.html + git add docs + + if [ -n "$(git status --porcelain --untracked-files=no)" ] + then + git commit -m "Update build metrics" + git push + fi diff --git a/.gitignore b/.gitignore index 243d56b4905a..614d10dc3f49 100644 --- a/.gitignore +++ b/.gitignore @@ -321,3 +321,4 @@ src/amalgamation/ #docs velox/docs/sphinx/source/README_generated_* velox/docs/bindings/python/_generate/* +scripts/bm-report/report.html diff --git a/scripts/bm-report/report.Rmd b/scripts/bm-report/report.Rmd index 9b1ca47544a2..d624d5a24c8e 100644 --- a/scripts/bm-report/report.Rmd +++ b/scripts/bm-report/report.Rmd @@ -154,19 +154,6 @@ size_formatter <- function() { gdata::humanReadable(x, standard = "Unix") } } -create_byte <- function(bytes) { - stopifnot(is.numeric(bytes)) - structure( - list(bytes = bytes), - class = "BytesObject" - ) -} -Bytes <- Vectorize(create_byte, SIMPLIFY = FALSE) - -print.BytesObject <- function(x, ...) { - human_readable <- prettyunits::pretty_bytes(x$bytes) - cat(human_readable) -} object_sizes <- results |> filter(endsWith(tags.source, "size"), commit.sha == newest_sha) diff --git a/scripts/bm-report/report.html b/scripts/bm-report/report.html deleted file mode 100644 index a6470f7a2ca6..000000000000 --- a/scripts/bm-report/report.html +++ /dev/null @@ -1,8658 +0,0 @@ - - - - - - - - - -Velox Build Metrics - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
- -
- -
-
-

Velox Build Metrics

-
- - - -
- - - - -
- - - -
- - -
- -
-
-
-
-
- -
-
-
-
-
- -
-
Compile Times - Release
-
-
-
- -
-
-
-
- -
-
Compile Times - Debug
-
-
-
- -
-
-
-
- -
-
Link Times - Release
-
-
-
- -
-
-
-
- -
-
Link Times - Debug
-
-
-
- -
-
-
-
-
-
-
-
- -
-
-
- -
-
Object Sizes - Release
-
-
-
- -
-
-
-
- -
-
Object Sizes - Debug
-
-
-
- -
-
-
-
-
-
- -
- - -
- - - - - \ No newline at end of file From aec95529c5d7eee65965fca7b3741df074b1bc5c Mon Sep 17 00:00:00 2001 From: Jacob Wujciak-Jens Date: Tue, 14 May 2024 23:57:58 +0200 Subject: [PATCH 3/9] use nix fix default.nix --- .github/workflows/build-metrics.yml | 34 +++------------- scripts/bm-report/default.nix | 61 +++++++++++++++++++++++++++++ scripts/bm-report/report.Rmd | 2 +- 3 files changed, 68 insertions(+), 29 deletions(-) create mode 100644 scripts/bm-report/default.nix diff --git a/.github/workflows/build-metrics.yml b/.github/workflows/build-metrics.yml index 1a581ea41bc1..dda6b6369f8a 100644 --- a/.github/workflows/build-metrics.yml +++ b/.github/workflows/build-metrics.yml @@ -134,44 +134,22 @@ jobs: git config --global user.email "velox@users.noreply.github.com" git config --global user.name "velox" - - uses: quarto-dev/quarto-actions/setup@c70b2f816b8d51a4badea651aee0602de0ad87bf - - uses: r-lib/actions/setup-r@fbafc3bc4ba114e72680c71e835c59b022606c46 - with: - install-r: false - use-public-rspm: true + - uses: cachix/install-nix-action@v26 - - name: Install Dependencies - env: - GITHUB_PAT: ${{ github.token }} - shell: Rscript {0} + - name: Build Environment run: | - install.packages( - c( - "dplyr", - "gdata", - "ggplot2", - "gh", - "gt", - "hms", - "jqr", - "jsonlite", - "lubridate", - "memoise", - "plotly", - "purrr", - "remotes" - )) - remotes::install_github("conbench/conbenchcoms") + cd scripts/bm-report + nix-build - name: Build Documentation env: CONBENCH_URL: "https://velox-conbench.voltrondata.run/" CONBENCH_EMAIL: "${{ secrets.CONBENCH_EMAIL }}" CONBENCH_PASSWORD: "${{ secrets.CONBENCH_PASSWORD }}" - run: | cd scripts/bm-report - quarto render report.Rmd + nix-shell --run "quarto render report.Rmd" + nix-shell --run "quarto --version" - name: Push Documentation # if: ${{ github.event_name == 'push' && github.repository == 'facebookincubator/velox'}} diff --git a/scripts/bm-report/default.nix b/scripts/bm-report/default.nix new file mode 100644 index 000000000000..4267be1044dd --- /dev/null +++ b/scripts/bm-report/default.nix @@ -0,0 +1,61 @@ +# This file was generated by the {rix} R package v0.6.0 on 2024-05-15 +# with following call: +# >rix::rix(r_ver = "abd6d48f8c77bea7dc51beb2adfa6ed3950d2585", +# > r_pkgs = c("dplyr", +# > "prettyunits", +# > "ggplot2", +# > "gh", +# > "gt", +# > "hms", +# > "jqr", +# > "jsonlite", +# > "lubridate", +# > "memoise", +# > "plotly", +# > "purrr", +# > "remotes"), +# > system_pkgs = c("quarto"), +# > git_pkgs = list(package_name = "conbenchcoms", +# > repo_url = "https://github.com/conbench/conbenchcoms", +# > branch_name = "main", +# > commit = "55cdb120bbe2c668d3cf8ae543f4922131653645"), +# > ide = "other", +# > project_path = path_default_nix, +# > overwrite = TRUE, +# > print = TRUE) +# It uses nixpkgs' revision abd6d48f8c77bea7dc51beb2adfa6ed3950d2585 for reproducibility purposes +# which will install R version latest +# Report any issues to https://github.com/b-rodrigues/rix +let + pkgs = import (fetchTarball "https://github.com/NixOS/nixpkgs/archive/abd6d48f8c77bea7dc51beb2adfa6ed3950d2585.tar.gz") {}; + rpkgs = builtins.attrValues { + inherit (pkgs.rPackages) dplyr prettyunits ggplot2 gh gt hms jqr jsonlite lubridate memoise plotly quarto purrr remotes; +}; + git_archive_pkgs = [(pkgs.rPackages.buildRPackage { + name = "conbenchcoms"; + src = pkgs.fetchgit { + url = "https://github.com/conbench/conbenchcoms"; + branchName = "main"; + rev = "55cdb120bbe2c668d3cf8ae543f4922131653645"; + sha256 = "sha256-XR5+grCUKyvSxsqiOkKd3gMUsWDJblZDmF4O+Jehq6U="; + }; + propagatedBuildInputs = builtins.attrValues { + inherit (pkgs.rPackages) dplyr glue httr2 yaml; + }; + }) ]; + system_packages = builtins.attrValues { + inherit (pkgs) R glibcLocales nix quartoMinimal; +}; + in + pkgs.mkShell { + LOCALE_ARCHIVE = if pkgs.system == "x86_64-linux" then "${pkgs.glibcLocales}/lib/locale/locale-archive" else ""; + LANG = "en_US.UTF-8"; + LC_ALL = "en_US.UTF-8"; + LC_TIME = "en_US.UTF-8"; + LC_MONETARY = "en_US.UTF-8"; + LC_PAPER = "en_US.UTF-8"; + LC_MEASUREMENT = "en_US.UTF-8"; + + buildInputs = [ git_archive_pkgs rpkgs system_packages ]; + + } diff --git a/scripts/bm-report/report.Rmd b/scripts/bm-report/report.Rmd index d624d5a24c8e..653d5f585f29 100644 --- a/scripts/bm-report/report.Rmd +++ b/scripts/bm-report/report.Rmd @@ -151,7 +151,7 @@ link_times |> ```{r big-objects} size_formatter <- function() { function(x) { - gdata::humanReadable(x, standard = "Unix") + prettyunits::pretty_bytes(x) } } From fc97ebd26035e10ee6b1bcd92096adaf51a01e5e Mon Sep 17 00:00:00 2001 From: Jacob Wujciak-Jens Date: Wed, 15 May 2024 00:08:58 +0200 Subject: [PATCH 4/9] remove no longer used package Revert "clean up" This reverts commit 2c7c81a92106d9b5139800bbdbb886151bce4f90. revert pretty --- .github/workflows/build-metrics.yml | 2 ++ scripts/bm-report/report.Rmd | 15 ++++++++++++++- 2 files changed, 16 insertions(+), 1 deletion(-) diff --git a/.github/workflows/build-metrics.yml b/.github/workflows/build-metrics.yml index dda6b6369f8a..24147418500a 100644 --- a/.github/workflows/build-metrics.yml +++ b/.github/workflows/build-metrics.yml @@ -135,6 +135,8 @@ jobs: git config --global user.name "velox" - uses: cachix/install-nix-action@v26 + with: + nix_path: nixpkgs=channel:nixos-unstable - name: Build Environment run: | diff --git a/scripts/bm-report/report.Rmd b/scripts/bm-report/report.Rmd index 653d5f585f29..9b1ca47544a2 100644 --- a/scripts/bm-report/report.Rmd +++ b/scripts/bm-report/report.Rmd @@ -151,9 +151,22 @@ link_times |> ```{r big-objects} size_formatter <- function() { function(x) { - prettyunits::pretty_bytes(x) + gdata::humanReadable(x, standard = "Unix") } } +create_byte <- function(bytes) { + stopifnot(is.numeric(bytes)) + structure( + list(bytes = bytes), + class = "BytesObject" + ) +} +Bytes <- Vectorize(create_byte, SIMPLIFY = FALSE) + +print.BytesObject <- function(x, ...) { + human_readable <- prettyunits::pretty_bytes(x$bytes) + cat(human_readable) +} object_sizes <- results |> filter(endsWith(tags.source, "size"), commit.sha == newest_sha) From 77a84bfb398600011ba228bc2a5dfbb4e81f29d1 Mon Sep 17 00:00:00 2001 From: Sam Albers Date: Wed, 15 May 2024 00:30:52 +0200 Subject: [PATCH 5/9] improved layout --- scripts/bm-report/report.Rmd | 137 +++++++++++++-- scripts/bm-report/report.qmd | 328 +++++++++++++++++++++++++++++++++++ 2 files changed, 450 insertions(+), 15 deletions(-) create mode 100644 scripts/bm-report/report.qmd diff --git a/scripts/bm-report/report.Rmd b/scripts/bm-report/report.Rmd index 9b1ca47544a2..5f33f63bf37b 100644 --- a/scripts/bm-report/report.Rmd +++ b/scripts/bm-report/report.Rmd @@ -1,8 +1,8 @@ --- title: "Velox Build Metrics" execute: - warning: false echo: false + warning: false format: html: grid: @@ -18,12 +18,16 @@ format: theme: cosmo --- + + + ```{r setup} library(gt) library(ggplot2) library(plotly) library(gdata) library(dplyr) +library(purrr) cd <- cachem::cache_disk(rappdirs::user_cache_dir("velox-bm-report")) mgh <- memoise::memoise(gh::gh, cache = cd) @@ -57,13 +61,30 @@ results <- run_ids |> ) ) ``` -::: {.panel-tabset} + +```{r ggplot2-specs} +theme_set(theme_minimal(base_size = 12) %+replace% + theme( + plot.title.position = "plot", + strip.text = element_text(size = 12) + )) + +format_tags <- function(x) { + x |> + stringr::str_replace_all("_", " ") |> + stringr::str_to_title() +} +``` + +::::: {.panel-tabset} + ## Times ```{r total-graphs} times_plot <- results |> filter(tags.suite == "total", endsWith(tags.source, "time")) |> mutate( stats.data = lubridate::dseconds(stats.data), + tags.name = format_tags(tags.name) ) |> ggplot(aes( x = timestamp, @@ -80,15 +101,27 @@ times_plot <- results |> x = "Date", y = "Time in Minutes" ) + - theme_minimal() -ggplotly(times_plot) + scale_color_viridis_d() +ggplotly(times_plot) |> + layout(legend = list(title = list(text = "Tags Name
"))) ## needed because theme legend specs don't work with ggplotly ``` -```{r expensive-objects} + +```{r expensive-objects-compile} compile_times <- results |> filter(tags.suite == "compiling", commit.sha == newest_sha) |> mutate( stats.data = lubridate::dseconds(stats.data), + tags.name = glue::glue("`{tags.name}`") ) +``` + +### Compile Times + +:::: {.columns} + +::: {.column width="49%"} + +```{r compile-times-release} compile_times |> filter(type == "release") |> select(tags.name, stats.data) |> @@ -99,9 +132,20 @@ compile_times |> `stats.data` = "Time" ) |> cols_align(align = "left", columns = everything()) |> - tab_header(title = "Compile Times - Release") |> + tab_header(title = "Release") |> + fmt_markdown(columns = "tags.name") |> opt_interactive(use_page_size_select = TRUE, use_search = TRUE) +``` + +::: + +::: {.column width="2%"} + +::: +::: {.column width="49%"} + +```{r compile-times-debug} compile_times |> filter(type == "debug") |> select(tags.name, stats.data) |> @@ -112,15 +156,32 @@ compile_times |> `stats.data` = "Time" ) |> cols_align(align = "left", columns = everything()) |> - tab_header(title = "Compile Times - Debug") |> + tab_header(title = "Debug") |> + fmt_markdown(columns = "tags.name") |> opt_interactive(use_page_size_select = TRUE, use_search = TRUE) +``` + +::: +:::: + +```{r expensive-objects-link} link_times <- results |> filter(tags.suite == "linking", commit.sha == newest_sha) |> mutate( stats.data = lubridate::dseconds(stats.data), + tags.name = glue::glue("`{tags.name}`") ) +``` + +### Link Times + +:::: {.columns} + +::: {.column width="49%"} + +```{r link-times-release} link_times |> filter(type == "release") |> select(tags.name, stats.data) |> @@ -131,9 +192,20 @@ link_times |> `stats.data` = "Time" ) |> cols_align(align = "left", columns = everything()) |> - tab_header(title = "Link Times - Release") |> + tab_header(title = "Release") |> + fmt_markdown(columns = "tags.name") |> opt_interactive(use_page_size_select = TRUE, use_search = TRUE) +``` +::: + +::: {.column width="2%"} + +::: + +::: {.column width="49%"} + +```{r link-times-debug} link_times |> filter(type == "debug") |> select(tags.name, stats.data) |> @@ -145,11 +217,18 @@ link_times |> ) |> cols_align(align = "left", columns = everything()) |> tab_header(title = "Link Times - Debug") |> + fmt_markdown(columns = "tags.name") |> opt_interactive(use_page_size_select = TRUE, use_search = TRUE) ``` + +::: + +:::: + + ## Sizes ```{r big-objects} -size_formatter <- function() { +size_formatter <- function(x) { function(x) { gdata::humanReadable(x, standard = "Unix") } @@ -169,7 +248,10 @@ print.BytesObject <- function(x, ...) { } object_sizes <- results |> - filter(endsWith(tags.source, "size"), commit.sha == newest_sha) + filter(endsWith(tags.source, "size"), commit.sha == newest_sha) |> + mutate( + tags.name = glue::glue("`{tags.name}`") + ) sizes_plot <- results |> filter(tags.suite == "executable", startsWith(tags.name, "total_")) |> ggplot(aes( @@ -180,16 +262,24 @@ sizes_plot <- results |> facet_wrap(~type) + geom_line() + geom_point() + - scale_y_continuous(labels = size_formatter()) + + # scale_y_continuous(labels = size_formatter()) + scale_x_datetime() + labs( title = "Velox Object Sizes", x = "Date", y = "Size" ) + - theme_minimal() -ggplotly(sizes_plot) + scale_color_viridis_d() +ggplotly(sizes_plot) |> + layout(legend = list(title = list(text = "Tags Name
"))) ## needed because theme legend specs don't work with ggplotly +``` + +### Object Sizes +:::: {.columns} +::: {.column width="49%"} + +```{r object-sizes-release} object_sizes |> filter(type == "release") |> select(tags.name, stats.data) |> @@ -200,22 +290,39 @@ object_sizes |> `stats.data` = "Size" ) |> fmt(columns = `stats.data`, fn = size_formatter()) |> + fmt_markdown(columns = "tags.name") |> cols_align(align = "left", columns = everything()) |> - tab_header(title = "Object Sizes - Release") |> + tab_header(title = "Release") |> opt_interactive(use_page_size_select = TRUE, use_search = TRUE) +``` + +::: +::: {.column width="2%"} + +::: + +::: {.column width="49%"} + +```{r object-sizes-debug} object_sizes |> filter(type == "debug") |> select(tags.name, stats.data) |> arrange(desc(stats.data)) |> gt() |> fmt(columns = `stats.data`, fn = size_formatter()) |> + fmt_markdown(columns = "tags.name") |> cols_label( `tags.name` = "Object", `stats.data` = "Time" ) |> cols_align(align = "left", columns = everything()) |> - tab_header(title = "Object Sizes - Debug") |> + tab_header(title = "Debug") |> opt_interactive(use_page_size_select = TRUE, use_search = TRUE) ``` + ::: + +:::: + +::::: diff --git a/scripts/bm-report/report.qmd b/scripts/bm-report/report.qmd new file mode 100644 index 000000000000..5f33f63bf37b --- /dev/null +++ b/scripts/bm-report/report.qmd @@ -0,0 +1,328 @@ +--- +title: "Velox Build Metrics" +execute: + echo: false + warning: false +format: + html: + grid: + sidebar-width: 0px + body-width: 1800px + margin-width: 150px + gutter-width: 1.5rem + self-contained: true + page-layout: full + toc: false + margin-left: 30px + link-external-newwindow: true + theme: cosmo +--- + + + + +```{r setup} +library(gt) +library(ggplot2) +library(plotly) +library(gdata) +library(dplyr) +library(purrr) + +cd <- cachem::cache_disk(rappdirs::user_cache_dir("velox-bm-report")) +mgh <- memoise::memoise(gh::gh, cache = cd) +mruns <- memoise::memoise(conbenchcoms::runs, cache = cd) +mresults <- memoise::memoise(conbenchcoms::benchmark_results, cache = cd) + +runs <- mgh( + "GET /repos/facebookincubator/velox/actions/workflows/build-metrics.yml/runs", + status = "success", + branch = "main" +) |> jsonlite::toJSON() + +newest_sha <- runs |> + jqr::jq(".workflow_runs | max_by(.updated_at) | .head_sha") |> + jsonlite::fromJSON() + +run_shas <- runs |> + jqr::jq("[.workflow_runs[].head_sha]") |> + jsonlite::fromJSON() +run_ids <- mruns(run_shas) |> + filter(commit.branch == "facebookincubator:main", substr(id, 1, 2) == "BM") |> + pull(id) +results <- run_ids |> + purrr::map_df(mresults) |> + mutate( + timestamp = lubridate::as_datetime(timestamp), + stats.data = unlist(stats.data), + type = case_when( + startsWith(run_id, "BM-debug") ~ "debug", + .default = "release" + ) + ) +``` + +```{r ggplot2-specs} +theme_set(theme_minimal(base_size = 12) %+replace% + theme( + plot.title.position = "plot", + strip.text = element_text(size = 12) + )) + +format_tags <- function(x) { + x |> + stringr::str_replace_all("_", " ") |> + stringr::str_to_title() +} +``` + +::::: {.panel-tabset} + +## Times +```{r total-graphs} +times_plot <- results |> + filter(tags.suite == "total", endsWith(tags.source, "time")) |> + mutate( + stats.data = lubridate::dseconds(stats.data), + tags.name = format_tags(tags.name) + ) |> + ggplot(aes( + x = timestamp, + y = stats.data, + group = interaction(tags.name, type), color = tags.name + )) + + facet_wrap(~type) + + geom_line() + + geom_point() + + scale_y_time() + + scale_x_datetime() + + labs( + title = "Velox Build Times", + x = "Date", + y = "Time in Minutes" + ) + + scale_color_viridis_d() +ggplotly(times_plot) |> + layout(legend = list(title = list(text = "Tags Name
"))) ## needed because theme legend specs don't work with ggplotly +``` + +```{r expensive-objects-compile} +compile_times <- results |> + filter(tags.suite == "compiling", commit.sha == newest_sha) |> + mutate( + stats.data = lubridate::dseconds(stats.data), + tags.name = glue::glue("`{tags.name}`") + ) +``` + +### Compile Times + +:::: {.columns} + +::: {.column width="49%"} + +```{r compile-times-release} +compile_times |> + filter(type == "release") |> + select(tags.name, stats.data) |> + arrange(desc(stats.data)) |> + gt() |> + cols_label( + `tags.name` = "Object", + `stats.data` = "Time" + ) |> + cols_align(align = "left", columns = everything()) |> + tab_header(title = "Release") |> + fmt_markdown(columns = "tags.name") |> + opt_interactive(use_page_size_select = TRUE, use_search = TRUE) +``` + +::: + +::: {.column width="2%"} + +::: + +::: {.column width="49%"} + +```{r compile-times-debug} +compile_times |> + filter(type == "debug") |> + select(tags.name, stats.data) |> + arrange(desc(stats.data)) |> + gt() |> + cols_label( + `tags.name` = "Object", + `stats.data` = "Time" + ) |> + cols_align(align = "left", columns = everything()) |> + tab_header(title = "Debug") |> + fmt_markdown(columns = "tags.name") |> + opt_interactive(use_page_size_select = TRUE, use_search = TRUE) +``` + +::: + +:::: + +```{r expensive-objects-link} +link_times <- results |> + filter(tags.suite == "linking", commit.sha == newest_sha) |> + mutate( + stats.data = lubridate::dseconds(stats.data), + tags.name = glue::glue("`{tags.name}`") + ) + +``` + +### Link Times + +:::: {.columns} + +::: {.column width="49%"} + +```{r link-times-release} +link_times |> + filter(type == "release") |> + select(tags.name, stats.data) |> + arrange(desc(stats.data)) |> + gt() |> + cols_label( + `tags.name` = "Object", + `stats.data` = "Time" + ) |> + cols_align(align = "left", columns = everything()) |> + tab_header(title = "Release") |> + fmt_markdown(columns = "tags.name") |> + opt_interactive(use_page_size_select = TRUE, use_search = TRUE) +``` + +::: + +::: {.column width="2%"} + +::: + +::: {.column width="49%"} + +```{r link-times-debug} +link_times |> + filter(type == "debug") |> + select(tags.name, stats.data) |> + arrange(desc(stats.data)) |> + gt() |> + cols_label( + `tags.name` = "Object", + `stats.data` = "Time" + ) |> + cols_align(align = "left", columns = everything()) |> + tab_header(title = "Link Times - Debug") |> + fmt_markdown(columns = "tags.name") |> + opt_interactive(use_page_size_select = TRUE, use_search = TRUE) +``` + +::: + +:::: + + +## Sizes +```{r big-objects} +size_formatter <- function(x) { + function(x) { + gdata::humanReadable(x, standard = "Unix") + } +} +create_byte <- function(bytes) { + stopifnot(is.numeric(bytes)) + structure( + list(bytes = bytes), + class = "BytesObject" + ) +} +Bytes <- Vectorize(create_byte, SIMPLIFY = FALSE) + +print.BytesObject <- function(x, ...) { + human_readable <- prettyunits::pretty_bytes(x$bytes) + cat(human_readable) +} + +object_sizes <- results |> + filter(endsWith(tags.source, "size"), commit.sha == newest_sha) |> + mutate( + tags.name = glue::glue("`{tags.name}`") + ) +sizes_plot <- results |> + filter(tags.suite == "executable", startsWith(tags.name, "total_")) |> + ggplot(aes( + x = timestamp, + y = stats.data, + group = interaction(tags.name, type), color = tags.name + )) + + facet_wrap(~type) + + geom_line() + + geom_point() + + # scale_y_continuous(labels = size_formatter()) + + scale_x_datetime() + + labs( + title = "Velox Object Sizes", + x = "Date", + y = "Size" + ) + + scale_color_viridis_d() +ggplotly(sizes_plot) |> + layout(legend = list(title = list(text = "Tags Name
"))) ## needed because theme legend specs don't work with ggplotly +``` + +### Object Sizes +:::: {.columns} + +::: {.column width="49%"} + +```{r object-sizes-release} +object_sizes |> + filter(type == "release") |> + select(tags.name, stats.data) |> + arrange(desc(stats.data)) |> + gt() |> + cols_label( + `tags.name` = "Object", + `stats.data` = "Size" + ) |> + fmt(columns = `stats.data`, fn = size_formatter()) |> + fmt_markdown(columns = "tags.name") |> + cols_align(align = "left", columns = everything()) |> + tab_header(title = "Release") |> + opt_interactive(use_page_size_select = TRUE, use_search = TRUE) +``` + +::: + +::: {.column width="2%"} + +::: + +::: {.column width="49%"} + +```{r object-sizes-debug} +object_sizes |> + filter(type == "debug") |> + select(tags.name, stats.data) |> + arrange(desc(stats.data)) |> + gt() |> + fmt(columns = `stats.data`, fn = size_formatter()) |> + fmt_markdown(columns = "tags.name") |> + cols_label( + `tags.name` = "Object", + `stats.data` = "Time" + ) |> + cols_align(align = "left", columns = everything()) |> + tab_header(title = "Debug") |> + opt_interactive(use_page_size_select = TRUE, use_search = TRUE) +``` + +::: + +:::: + +::::: From c892266b4e6d11c24e0cdf4bc30fb41c3ca170ea Mon Sep 17 00:00:00 2001 From: Jacob Wujciak-Jens Date: Wed, 15 May 2024 00:34:41 +0200 Subject: [PATCH 6/9] use pretty units instead of gdata --- scripts/bm-report/report.Rmd | 18 ++---------------- 1 file changed, 2 insertions(+), 16 deletions(-) diff --git a/scripts/bm-report/report.Rmd b/scripts/bm-report/report.Rmd index 5f33f63bf37b..75ab872deee3 100644 --- a/scripts/bm-report/report.Rmd +++ b/scripts/bm-report/report.Rmd @@ -25,7 +25,6 @@ format: library(gt) library(ggplot2) library(plotly) -library(gdata) library(dplyr) library(purrr) @@ -230,22 +229,9 @@ link_times |> ```{r big-objects} size_formatter <- function(x) { function(x) { - gdata::humanReadable(x, standard = "Unix") + prettyunits::pretty_bytes(x) } } -create_byte <- function(bytes) { - stopifnot(is.numeric(bytes)) - structure( - list(bytes = bytes), - class = "BytesObject" - ) -} -Bytes <- Vectorize(create_byte, SIMPLIFY = FALSE) - -print.BytesObject <- function(x, ...) { - human_readable <- prettyunits::pretty_bytes(x$bytes) - cat(human_readable) -} object_sizes <- results |> filter(endsWith(tags.source, "size"), commit.sha == newest_sha) |> @@ -262,7 +248,7 @@ sizes_plot <- results |> facet_wrap(~type) + geom_line() + geom_point() + - # scale_y_continuous(labels = size_formatter()) + + scale_y_continuous(labels = size_formatter()) + scale_x_datetime() + labs( title = "Velox Object Sizes", From df5e2a69e58d78b5e7e966ea5d185f59165731c9 Mon Sep 17 00:00:00 2001 From: Jacob Wujciak-Jens Date: Wed, 15 May 2024 00:35:17 +0200 Subject: [PATCH 7/9] use correct extension --- .github/workflows/build-metrics.yml | 3 +- scripts/bm-report/report.Rmd | 314 ---------------------------- scripts/bm-report/report.qmd | 18 +- 3 files changed, 3 insertions(+), 332 deletions(-) delete mode 100644 scripts/bm-report/report.Rmd diff --git a/.github/workflows/build-metrics.yml b/.github/workflows/build-metrics.yml index 24147418500a..5d616e2fdf39 100644 --- a/.github/workflows/build-metrics.yml +++ b/.github/workflows/build-metrics.yml @@ -150,8 +150,7 @@ jobs: CONBENCH_PASSWORD: "${{ secrets.CONBENCH_PASSWORD }}" run: | cd scripts/bm-report - nix-shell --run "quarto render report.Rmd" - nix-shell --run "quarto --version" + nix-shell --run "quarto render report.qmd" - name: Push Documentation # if: ${{ github.event_name == 'push' && github.repository == 'facebookincubator/velox'}} diff --git a/scripts/bm-report/report.Rmd b/scripts/bm-report/report.Rmd deleted file mode 100644 index 75ab872deee3..000000000000 --- a/scripts/bm-report/report.Rmd +++ /dev/null @@ -1,314 +0,0 @@ ---- -title: "Velox Build Metrics" -execute: - echo: false - warning: false -format: - html: - grid: - sidebar-width: 0px - body-width: 1800px - margin-width: 150px - gutter-width: 1.5rem - self-contained: true - page-layout: full - toc: false - margin-left: 30px - link-external-newwindow: true - theme: cosmo ---- - - - - -```{r setup} -library(gt) -library(ggplot2) -library(plotly) -library(dplyr) -library(purrr) - -cd <- cachem::cache_disk(rappdirs::user_cache_dir("velox-bm-report")) -mgh <- memoise::memoise(gh::gh, cache = cd) -mruns <- memoise::memoise(conbenchcoms::runs, cache = cd) -mresults <- memoise::memoise(conbenchcoms::benchmark_results, cache = cd) - -runs <- mgh( - "GET /repos/facebookincubator/velox/actions/workflows/build-metrics.yml/runs", - status = "success", - branch = "main" -) |> jsonlite::toJSON() - -newest_sha <- runs |> - jqr::jq(".workflow_runs | max_by(.updated_at) | .head_sha") |> - jsonlite::fromJSON() - -run_shas <- runs |> - jqr::jq("[.workflow_runs[].head_sha]") |> - jsonlite::fromJSON() -run_ids <- mruns(run_shas) |> - filter(commit.branch == "facebookincubator:main", substr(id, 1, 2) == "BM") |> - pull(id) -results <- run_ids |> - purrr::map_df(mresults) |> - mutate( - timestamp = lubridate::as_datetime(timestamp), - stats.data = unlist(stats.data), - type = case_when( - startsWith(run_id, "BM-debug") ~ "debug", - .default = "release" - ) - ) -``` - -```{r ggplot2-specs} -theme_set(theme_minimal(base_size = 12) %+replace% - theme( - plot.title.position = "plot", - strip.text = element_text(size = 12) - )) - -format_tags <- function(x) { - x |> - stringr::str_replace_all("_", " ") |> - stringr::str_to_title() -} -``` - -::::: {.panel-tabset} - -## Times -```{r total-graphs} -times_plot <- results |> - filter(tags.suite == "total", endsWith(tags.source, "time")) |> - mutate( - stats.data = lubridate::dseconds(stats.data), - tags.name = format_tags(tags.name) - ) |> - ggplot(aes( - x = timestamp, - y = stats.data, - group = interaction(tags.name, type), color = tags.name - )) + - facet_wrap(~type) + - geom_line() + - geom_point() + - scale_y_time() + - scale_x_datetime() + - labs( - title = "Velox Build Times", - x = "Date", - y = "Time in Minutes" - ) + - scale_color_viridis_d() -ggplotly(times_plot) |> - layout(legend = list(title = list(text = "Tags Name
"))) ## needed because theme legend specs don't work with ggplotly -``` - -```{r expensive-objects-compile} -compile_times <- results |> - filter(tags.suite == "compiling", commit.sha == newest_sha) |> - mutate( - stats.data = lubridate::dseconds(stats.data), - tags.name = glue::glue("`{tags.name}`") - ) -``` - -### Compile Times - -:::: {.columns} - -::: {.column width="49%"} - -```{r compile-times-release} -compile_times |> - filter(type == "release") |> - select(tags.name, stats.data) |> - arrange(desc(stats.data)) |> - gt() |> - cols_label( - `tags.name` = "Object", - `stats.data` = "Time" - ) |> - cols_align(align = "left", columns = everything()) |> - tab_header(title = "Release") |> - fmt_markdown(columns = "tags.name") |> - opt_interactive(use_page_size_select = TRUE, use_search = TRUE) -``` - -::: - -::: {.column width="2%"} - -::: - -::: {.column width="49%"} - -```{r compile-times-debug} -compile_times |> - filter(type == "debug") |> - select(tags.name, stats.data) |> - arrange(desc(stats.data)) |> - gt() |> - cols_label( - `tags.name` = "Object", - `stats.data` = "Time" - ) |> - cols_align(align = "left", columns = everything()) |> - tab_header(title = "Debug") |> - fmt_markdown(columns = "tags.name") |> - opt_interactive(use_page_size_select = TRUE, use_search = TRUE) -``` - -::: - -:::: - -```{r expensive-objects-link} -link_times <- results |> - filter(tags.suite == "linking", commit.sha == newest_sha) |> - mutate( - stats.data = lubridate::dseconds(stats.data), - tags.name = glue::glue("`{tags.name}`") - ) - -``` - -### Link Times - -:::: {.columns} - -::: {.column width="49%"} - -```{r link-times-release} -link_times |> - filter(type == "release") |> - select(tags.name, stats.data) |> - arrange(desc(stats.data)) |> - gt() |> - cols_label( - `tags.name` = "Object", - `stats.data` = "Time" - ) |> - cols_align(align = "left", columns = everything()) |> - tab_header(title = "Release") |> - fmt_markdown(columns = "tags.name") |> - opt_interactive(use_page_size_select = TRUE, use_search = TRUE) -``` - -::: - -::: {.column width="2%"} - -::: - -::: {.column width="49%"} - -```{r link-times-debug} -link_times |> - filter(type == "debug") |> - select(tags.name, stats.data) |> - arrange(desc(stats.data)) |> - gt() |> - cols_label( - `tags.name` = "Object", - `stats.data` = "Time" - ) |> - cols_align(align = "left", columns = everything()) |> - tab_header(title = "Link Times - Debug") |> - fmt_markdown(columns = "tags.name") |> - opt_interactive(use_page_size_select = TRUE, use_search = TRUE) -``` - -::: - -:::: - - -## Sizes -```{r big-objects} -size_formatter <- function(x) { - function(x) { - prettyunits::pretty_bytes(x) - } -} - -object_sizes <- results |> - filter(endsWith(tags.source, "size"), commit.sha == newest_sha) |> - mutate( - tags.name = glue::glue("`{tags.name}`") - ) -sizes_plot <- results |> - filter(tags.suite == "executable", startsWith(tags.name, "total_")) |> - ggplot(aes( - x = timestamp, - y = stats.data, - group = interaction(tags.name, type), color = tags.name - )) + - facet_wrap(~type) + - geom_line() + - geom_point() + - scale_y_continuous(labels = size_formatter()) + - scale_x_datetime() + - labs( - title = "Velox Object Sizes", - x = "Date", - y = "Size" - ) + - scale_color_viridis_d() -ggplotly(sizes_plot) |> - layout(legend = list(title = list(text = "Tags Name
"))) ## needed because theme legend specs don't work with ggplotly -``` - -### Object Sizes -:::: {.columns} - -::: {.column width="49%"} - -```{r object-sizes-release} -object_sizes |> - filter(type == "release") |> - select(tags.name, stats.data) |> - arrange(desc(stats.data)) |> - gt() |> - cols_label( - `tags.name` = "Object", - `stats.data` = "Size" - ) |> - fmt(columns = `stats.data`, fn = size_formatter()) |> - fmt_markdown(columns = "tags.name") |> - cols_align(align = "left", columns = everything()) |> - tab_header(title = "Release") |> - opt_interactive(use_page_size_select = TRUE, use_search = TRUE) -``` - -::: - -::: {.column width="2%"} - -::: - -::: {.column width="49%"} - -```{r object-sizes-debug} -object_sizes |> - filter(type == "debug") |> - select(tags.name, stats.data) |> - arrange(desc(stats.data)) |> - gt() |> - fmt(columns = `stats.data`, fn = size_formatter()) |> - fmt_markdown(columns = "tags.name") |> - cols_label( - `tags.name` = "Object", - `stats.data` = "Time" - ) |> - cols_align(align = "left", columns = everything()) |> - tab_header(title = "Debug") |> - opt_interactive(use_page_size_select = TRUE, use_search = TRUE) -``` - -::: - -:::: - -::::: diff --git a/scripts/bm-report/report.qmd b/scripts/bm-report/report.qmd index 5f33f63bf37b..75ab872deee3 100644 --- a/scripts/bm-report/report.qmd +++ b/scripts/bm-report/report.qmd @@ -25,7 +25,6 @@ format: library(gt) library(ggplot2) library(plotly) -library(gdata) library(dplyr) library(purrr) @@ -230,22 +229,9 @@ link_times |> ```{r big-objects} size_formatter <- function(x) { function(x) { - gdata::humanReadable(x, standard = "Unix") + prettyunits::pretty_bytes(x) } } -create_byte <- function(bytes) { - stopifnot(is.numeric(bytes)) - structure( - list(bytes = bytes), - class = "BytesObject" - ) -} -Bytes <- Vectorize(create_byte, SIMPLIFY = FALSE) - -print.BytesObject <- function(x, ...) { - human_readable <- prettyunits::pretty_bytes(x$bytes) - cat(human_readable) -} object_sizes <- results |> filter(endsWith(tags.source, "size"), commit.sha == newest_sha) |> @@ -262,7 +248,7 @@ sizes_plot <- results |> facet_wrap(~type) + geom_line() + geom_point() + - # scale_y_continuous(labels = size_formatter()) + + scale_y_continuous(labels = size_formatter()) + scale_x_datetime() + labs( title = "Velox Object Sizes", From 04c010f379e66260d861f19d1868290dd545e09a Mon Sep 17 00:00:00 2001 From: Jacob Wujciak-Jens Date: Wed, 15 May 2024 00:44:19 +0200 Subject: [PATCH 8/9] uncomment `needs` etc. --- .github/workflows/build-metrics.yml | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/.github/workflows/build-metrics.yml b/.github/workflows/build-metrics.yml index 5d616e2fdf39..366f7a4e5e9d 100644 --- a/.github/workflows/build-metrics.yml +++ b/.github/workflows/build-metrics.yml @@ -122,7 +122,7 @@ jobs: contents: write runs-on: ubuntu-latest name: Generate and Upload Build Metric Report - # needs: metrics + needs: metrics steps: - name: Checkout uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4.1.1 @@ -134,7 +134,7 @@ jobs: git config --global user.email "velox@users.noreply.github.com" git config --global user.name "velox" - - uses: cachix/install-nix-action@v26 + - uses: cachix/install-nix-action@8887e596b4ee1134dae06b98d573bd674693f47c # v26 with: nix_path: nixpkgs=channel:nixos-unstable @@ -152,8 +152,10 @@ jobs: cd scripts/bm-report nix-shell --run "quarto render report.qmd" - - name: Push Documentation - # if: ${{ github.event_name == 'push' && github.repository == 'facebookincubator/velox'}} + - name: Push Report + # The report only uses conbench data from 'main' + # so any data generated in a PR won't be included + if: ${{ github.event_name != 'pull_request' && github.repository == 'facebookincubator/velox'}} run: | git checkout gh-pages mkdir -p docs/bm-report From 0b69693f63c8d0b3014a21a2e397234442380ebc Mon Sep 17 00:00:00 2001 From: Jacob Wujciak-Jens Date: Fri, 24 May 2024 05:30:16 +0200 Subject: [PATCH 9/9] add comments to report.qmd --- scripts/bm-report/report.qmd | 19 +++++++++++++++++++ 1 file changed, 19 insertions(+) diff --git a/scripts/bm-report/report.qmd b/scripts/bm-report/report.qmd index 75ab872deee3..9951343deec0 100644 --- a/scripts/bm-report/report.qmd +++ b/scripts/bm-report/report.qmd @@ -28,17 +28,21 @@ library(plotly) library(dplyr) library(purrr) +# Cache conbench and gh api results for local development cd <- cachem::cache_disk(rappdirs::user_cache_dir("velox-bm-report")) mgh <- memoise::memoise(gh::gh, cache = cd) mruns <- memoise::memoise(conbenchcoms::runs, cache = cd) mresults <- memoise::memoise(conbenchcoms::benchmark_results, cache = cd) +# Get latest runs of build-metric job runs <- mgh( "GET /repos/facebookincubator/velox/actions/workflows/build-metrics.yml/runs", status = "success", branch = "main" ) |> jsonlite::toJSON() +# Extract the commit sha of the most recent run. The results of the latest +# run are displayed in the tables. newest_sha <- runs |> jqr::jq(".workflow_runs | max_by(.updated_at) | .head_sha") |> jsonlite::fromJSON() @@ -49,6 +53,8 @@ run_shas <- runs |> run_ids <- mruns(run_shas) |> filter(commit.branch == "facebookincubator:main", substr(id, 1, 2) == "BM") |> pull(id) + +# Fetch the result and do clean/format the data results <- run_ids |> purrr::map_df(mresults) |> mutate( @@ -79,6 +85,7 @@ format_tags <- function(x) { ## Times ```{r total-graphs} +# Filter the data and layout the overview plots times_plot <- results |> filter(tags.suite == "total", endsWith(tags.source, "time")) |> mutate( @@ -106,6 +113,7 @@ ggplotly(times_plot) |> ``` ```{r expensive-objects-compile} +# Format compile time data compile_times <- results |> filter(tags.suite == "compiling", commit.sha == newest_sha) |> mutate( @@ -121,6 +129,7 @@ compile_times <- results |> ::: {.column width="49%"} ```{r compile-times-release} +# Select and format the data to be displayed in the release compile time table compile_times |> filter(type == "release") |> select(tags.name, stats.data) |> @@ -145,6 +154,7 @@ compile_times |> ::: {.column width="49%"} ```{r compile-times-debug} +# Select and format the data to be displayed in the debug compile time table compile_times |> filter(type == "debug") |> select(tags.name, stats.data) |> @@ -165,6 +175,7 @@ compile_times |> :::: ```{r expensive-objects-link} +# Format linke time data link_times <- results |> filter(tags.suite == "linking", commit.sha == newest_sha) |> mutate( @@ -181,6 +192,7 @@ link_times <- results |> ::: {.column width="49%"} ```{r link-times-release} +# Select and format the data to be displayed in the release link time table link_times |> filter(type == "release") |> select(tags.name, stats.data) |> @@ -205,6 +217,7 @@ link_times |> ::: {.column width="49%"} ```{r link-times-debug} +# Select and format the data to be displayed in the debug link time table link_times |> filter(type == "debug") |> select(tags.name, stats.data) |> @@ -227,17 +240,21 @@ link_times |> ## Sizes ```{r big-objects} +# This is converts byte values into human-readable values in the tables size_formatter <- function(x) { function(x) { prettyunits::pretty_bytes(x) } } +# Prepare object size data object_sizes <- results |> filter(endsWith(tags.source, "size"), commit.sha == newest_sha) |> mutate( tags.name = glue::glue("`{tags.name}`") ) + +# Filter the data and layout the size overview plots sizes_plot <- results |> filter(tags.suite == "executable", startsWith(tags.name, "total_")) |> ggplot(aes( @@ -266,6 +283,7 @@ ggplotly(sizes_plot) |> ::: {.column width="49%"} ```{r object-sizes-release} +# Select and format the data to be displayed in the release size table object_sizes |> filter(type == "release") |> select(tags.name, stats.data) |> @@ -291,6 +309,7 @@ object_sizes |> ::: {.column width="49%"} ```{r object-sizes-debug} +# Select and format the data to be displayed in the debug size table object_sizes |> filter(type == "debug") |> select(tags.name, stats.data) |>