diff --git a/.github/workflows/benchmark.yml b/.github/workflows/benchmark.yml index 4ff826df7ad74..dbbf25cc6f55b 100644 --- a/.github/workflows/benchmark.yml +++ b/.github/workflows/benchmark.yml @@ -22,6 +22,8 @@ on: - 'third_party/**' - 'pyvelox/**' - '.github/workflows/benchmark.yml' + - 'scripts/benchmark-requirements.txt' + push: branches: [main] diff --git a/.github/workflows/build-metrics.yml b/.github/workflows/build-metrics.yml new file mode 100644 index 0000000000000..c493d70c0f0c3 --- /dev/null +++ b/.github/workflows/build-metrics.yml @@ -0,0 +1,118 @@ +# Copyright (c) Facebook, Inc. and its affiliates. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +name: Collect Build Metrics + +on: + pull_request: + paths: + - ".github/workflows/build-metrics.yml" + + workflow_dispatch: + inputs: + ref: + description: "ref to check" + required: true + + schedule: + # Run every day at 04:05 + - cron: "5 4 * * *" + +permissions: + contents: read + +jobs: + metrics: + name: Linux ${{ matrix.type }} with adapters + if: ${{ github.repository == 'facebookincubator/velox' }} + runs-on: ${{ matrix.runner }} + container: ghcr.io/facebookincubator/velox-dev:adapters + strategy: + fail-fast: false + matrix: + - runner: "16-core" + - type: ["debug", "release"] + defaults: + run: + shell: bash + env: + VELOX_DEPENDENCY_SOURCE: SYSTEM + simdjson_SOURCE: BUNDLED + xsimd_SOURCE: BUNDLED + steps: + - uses: actions/checkout@v4 + with: + ref: ${{ inputs.ref || github.sha }} + + - name: Fix git permissions + # Usually actions/checkout does this but as we run in a container + # it doesn't work + run: git config --global --add safe.directory /__w/velox/velox + + - name: Make ${{ matrix.type }} Build + env: + MAKEFLAGS: 'MAX_HIGH_MEM_JOBS=8 MAX_LINK_JOBS=4' + run: | + EXTRA_CMAKE_FLAGS=( + "-DVELOX_ENABLE_BENCHMARKS=ON" + "-DVELOX_ENABLE_ARROW=ON" + "-DVELOX_ENABLE_PARQUET=ON" + "-DVELOX_ENABLE_HDFS=ON" + "-DVELOX_ENABLE_S3=ON" + "-DVELOX_ENABLE_GCS=ON" + "-DVELOX_ENABLE_ABFS=ON" + "-DVELOX_ENABLE_REMOTE_FUNCTIONS=ON" + ) + make '${{ matrix.type }}' + + - name: Log binary sizes + run: | + mkdir -p /tmp/metrics + sizes_file=/tmp/metrics/object_sizes + pushd '_build/${{ matrix.type }}' + + find velox -type f -name '*.so' -o -name '*.a' -exec ls -l -BB {} \; | + awk '{print $5, $9; total += $5} END {print total," total_lib_size"}' > $sizes_file + + find velox -type f -name '*.o' -exec ls -l -BB {} \; | + awk '{print $5, $9; total += $5} END {print total," total_obj_size"}' >> $sizes_file + + find velox -type f -name 'velox_*' -exec ls -l -BB {} \; | + awk '{print $5, $9; total += $5} END {print total," total_exec_size"}' >> $sizes_file + + - name: Copy ninja_log + run: cp _build/${{ matrix.type }}/.ninja_log /tmp/metrics/.ninja_log + + - name: "Install dependencies" + run: | + python3 -m pip install setuptools + python3 -m pip install -r scripts/benchmark-requirements.txt + + - name: "Upload Metrics" + env: + CONBENCH_URL: "https://velox-conbench.voltrondata.run/" + CONBENCH_MACHINE_INFO_NAME: "GitHub-runner-${{ matrix.runner }}" + CONBENCH_EMAIL: "${{ secrets.CONBENCH_EMAIL }}" + CONBENCH_PASSWORD: "${{ secrets.CONBENCH_PASSWORD }}" + # These don't actually work https://github.com/conbench/conbench/issues/1484 + # but have to be there to work regardless?? + CONBENCH_PROJECT_REPOSITORY: "${{ github.repository }}" + CONBENCH_PROJECT_COMMIT: "${{ inputs.ref || github.sha }}" + run: | + ./scripts/build-metrics.py upload \ + --build_type "${{ matrix.type }}" \ + --run_id "BM-${{ matrix.type }}-${{ github.run_id }}-${{ github.run_attempt }}" \ + --pr_number "${{ github.event.number }}" \ + --sha "${{ inputs.ref || github.sha }}" \ + "/tmp/metrics" diff --git a/.github/workflows/linux-build.yml b/.github/workflows/linux-build.yml index 6c547b35f5488..e8a68eda92f5b 100644 --- a/.github/workflows/linux-build.yml +++ b/.github/workflows/linux-build.yml @@ -104,7 +104,6 @@ jobs: "-DVELOX_ENABLE_S3=ON" "-DVELOX_ENABLE_GCS=ON" "-DVELOX_ENABLE_ABFS=ON" - "-DVELOX_ENABLE_SUBSTRAIT=ON" "-DVELOX_ENABLE_REMOTE_FUNCTIONS=ON" "-DVELOX_ENABLE_GPU=ON" ) diff --git a/scripts/benchmark-requirements.txt b/scripts/benchmark-requirements.txt index 7ae4d80284615..3df0a15d0bd45 100644 --- a/scripts/benchmark-requirements.txt +++ b/scripts/benchmark-requirements.txt @@ -12,6 +12,6 @@ # See the License for the specific language governing permissions and # limitations under the License. -benchadapt@git+https://github.com/conbench/conbench.git@44e81d1#subdirectory=benchadapt/python -benchalerts@git+https://github.com/conbench/conbench.git@44e81d1#subdirectory=benchalerts -benchclients@git+https://github.com/conbench/conbench.git@44e81d1#subdirectory=benchclients/python +benchadapt==2024.3.20 +benchalerts==2024.1.10.1 +benchclients==2024.3.29.1 diff --git a/scripts/build-metrics.py b/scripts/build-metrics.py new file mode 100755 index 0000000000000..d172a03708b36 --- /dev/null +++ b/scripts/build-metrics.py @@ -0,0 +1,295 @@ +#!/usr/bin/env python3 +# Copyright (c) Facebook, Inc. and its affiliates. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import argparse +import sys +import uuid +from os.path import join, splitext +from pathlib import Path +from typing import Any, Dict, List + +from benchadapt import BenchmarkResult +from benchadapt.adapters import BenchmarkAdapter + + +class BinarySizeAdapter(BenchmarkAdapter): + """ + Adapter to track build artifact sizes in conbench. + Expects the `size_file` to be formatted like this: + + + Suite meta data will be library, object or executable + based on file ending. + """ + + size_file: Path + + def __init__( + self, + command: List[str], + size_file: str, + build_type: str, + result_fields_override: Dict[str, Any] = {}, + result_fields_append: Dict[str, Any] = {}, + ) -> None: + self.size_file = Path(size_file) + if build_type not in ["debug", "release"]: + raise ValueError(f"Build type '{build_type}' is not valid!") + self.build_type = build_type + super().__init__(command, result_fields_override, result_fields_append) + + def _transform_results(self) -> List[BenchmarkResult]: + results = [] + + batch_id = uuid.uuid4().hex + with open(self.size_file, "r") as file: + sizes = [line.strip() for line in file] + + if not sizes: + raise ValueError("'size_file' is empty!") + + for line in sizes: + size, path = line.split(maxsplit=1) + path = path.strip() + _, ext = splitext(path) + if ext in [".so", ".a"]: + suite = "library" + elif ext == ".o": + suite = "object" + else: + suite = "executable" + + parsed_size = BenchmarkResult( + run_reason="merge", + batch_id=batch_id, + stats={ + "data": [size], + "unit": "B", + "iterations": 1, + }, + tags={ + "name": path, + "suite": suite, + "source": f"{self.build_type}_build_metrics_size", + }, + info={}, + context={"benchmark_language": "C++"}, + ) + results.append(parsed_size) + + return results + + +class NinjaLogAdapter(BenchmarkAdapter): + """ + Adapter to extract compile and link times from a .ninja_log. + Will calculate aggregates for total, compile, link and wall time. + Suite metadata will be set based on binary ending to object, library or executable. + + Only files in paths beginning with velox/ will be tracked to avoid dependencies. + """ + + ninja_log: Path + + def __init__( + self, + command: List[str], + ninja_log: str, + build_type: str, + result_fields_override: Dict[str, Any] = {}, + result_fields_append: Dict[str, Any] = {}, + ) -> None: + self.ninja_log = Path(ninja_log) + if build_type not in ["debug", "release"]: + raise ValueError(f"Build type '{build_type}' is not valid!") + self.build_type = build_type + super().__init__(command, result_fields_override, result_fields_append) + + def _transform_results(self) -> List[BenchmarkResult]: + results = [] + + batch_id = uuid.uuid4().hex + with open(self.ninja_log, "r") as file: + log_lines = [line.strip() for line in file] + + if not log_lines[0].startswith("# ninja log v"): + raise ValueError("Malformed Ninja log found!") + else: + del log_lines[0] + + ms2sec = lambda x: x / 1000 + get_epoch = lambda l: int(l.split()[2]) + totals = { + "link_time": 0, + "compile_time": 0, + "total_time": 0, + "wall_time": get_epoch(log_lines[-1]) - get_epoch(log_lines[0]), + } + + for line in log_lines: + start, end, epoch, object_path, _ = line.split() + start = int(start) + end = int(end) + duration = ms2sec(end - start) + + # Don't track dependency times (refine check potentially?) + if not object_path.startswith("velox"): + continue + + _, ext = splitext(object_path) + if ext in [".so", ".a"] or not ext: + totals["link_time"] += duration + suite = "linking" + elif ext == ".o": + totals["compile_time"] += duration + suite = "compiling" + else: + print(f"Unkown file type found: {object_path}") + print("Skipping...") + continue + + time_result = BenchmarkResult( + run_reason="merge", + batch_id=batch_id, + stats={ + "data": [duration], + "unit": "s", + "iterations": 1, + }, + tags={ + "name": object_path, + "suite": suite, + "source": f"{self.build_type}_build_metrics_time", + }, + info={}, + context={"benchmark_language": "C++"}, + ) + results.append(time_result) + + totals["total_time"] = totals["link_time"] + totals["compile_time"] + for total_name, total in totals.items(): + total_result = BenchmarkResult( + run_reason="merge", + batch_id=batch_id, + stats={ + "data": [total], + "unit": "s", + "iterations": 1, + }, + tags={ + "name": total_name, + "suite": "total", + "source": f"{self.build_type}_build_metrics_time", + }, + info={}, + context={"benchmark_language": "C++"}, + ) + results.append(total_result) + + return results + + +# find velox -type f -name '*.o' -exec ls -l -BB {} \; | awk '{print $5, $9}' | sed 's|CMakeFiles/.*dir/||g' > /tmp/object-size + + +def upload(args): + print("Uploading Build Metrics") + pr_number = int(args.pr_number) if args.pr_number else None + run_reason = "pull request" if pr_number else "commit" + run_name = f"{run_reason}: {args.sha}" + sizes = BinarySizeAdapter( + command=["true"], + size_file=join(args.base_path, args.size_file), + build_type=args.build_type, + result_fields_override={ + "run_id": args.run_id, + "run_name": run_name, + "run_reason": run_reason, + "github": { + "repository": "https://github.com/facebookincubator/velox", + "pr_number": pr_number, + "commit": args.sha, + }, + }, + ) + sizes() + + times = NinjaLogAdapter( + command=["true"], + ninja_log=join(args.base_path, args.ninja_log), + build_type=args.build_type, + result_fields_override={ + "run_id": args.run_id, + "run_name": run_name, + "run_reason": run_reason, + "github": { + "repository": "https://github.com/facebookincubator/velox", + "pr_number": pr_number, + "commit": args.sha, + }, + }, + ) + times() + + +def parse_args(): + parser = argparse.ArgumentParser(description="Velox Build Metric Utility.") + parser.set_defaults(func=lambda _: parser.print_help()) + + subparsers = parser.add_subparsers(help="Please specify one of the subcommands.") + + upload_parser = subparsers.add_parser( + "upload", help="Parse and upload build metrics" + ) + upload_parser.set_defaults(func=upload) + upload_parser.add_argument( + "--ninja_log", default=".ninja_log", help="Name of the ninja log file." + ) + upload_parser.add_argument( + "--size_file", + default="object_sizes", + help="Name of the file containing size information.", + ) + upload_parser.add_argument( + "--build_type", + required=True, + help="Type of build results come from, e.g. debug or release", + ) + upload_parser.add_argument( + "--run_id", + required=True, + help="A Conbench run ID unique to this build.", + ) + upload_parser.add_argument( + "--sha", + required=True, + help="HEAD sha for the result upload to conbench.", + ) + upload_parser.add_argument( + "--pr_number", + default=0, + help="PR number for the result upload to conbench.", + ) + upload_parser.add_argument( + "base_path", + help="Path in which the .ninja_log and sizes_file are found.", + ) + + return parser.parse_args() + + +if __name__ == "__main__": + args = parse_args() + args.func(args)